def create_measurements_dict(self): measurements_record = {} for each_channel, each_measurement in self.measurement.values.items(): measurement = self.device_measurements.filter( DeviceMeasurements.channel == each_channel).first() if measurement and 'value' in each_measurement: conversion = self.conversions.filter( Conversion.unique_id == measurement.conversion_id).first() # If a timestamp is passed from the module, use it if 'timestamp_utc' in each_measurement: timestamp = each_measurement['timestamp_utc'] else: timestamp = None measurements_record = parse_measurement( conversion, measurement, measurements_record, each_channel, each_measurement, timestamp=timestamp) self.logger.debug( f"Adding measurements to InfluxDB with ID {self.unique_id}: {measurements_record}") return measurements_record
def add_measurement_influxdb(self, channel, measurement): # Convert value/unit is conversion_id present and valid if self.channels_conversion[channel]: conversion = db_retrieve_table_daemon( Conversion, unique_id=self.channels_measurement[channel].conversion_id) if conversion: meas = parse_measurement( self.channels_conversion[channel], self.channels_measurement[channel], measurement, channel, measurement[channel], timestamp=measurement[channel]['timestamp_utc']) measurement[channel]['measurement'] = meas[channel][ 'measurement'] measurement[channel]['unit'] = meas[channel]['unit'] measurement[channel]['value'] = meas[channel]['value'] if measurement: self.logger.debug( "Adding measurement to influxdb: {}".format(measurement)) add_measurements_influxdb(self.unique_id, measurement, use_same_timestamp=INPUT_INFORMATION[ 'measurements_use_same_timestamp'])
def loop(self): if self.timer_loop > time.time(): return while self.timer_loop < time.time(): self.timer_loop += self.period measurements = {} for channel in self.channels_measurement: # Original value/unit measurements[channel] = {} measurements[channel]['measurement'] = self.channels_measurement[ channel].measurement measurements[channel]['unit'] = self.channels_measurement[ channel].unit measurements[channel]['value'] = random.randint(1, 100) # Convert value/unit is conversion_id present and valid if self.channels_conversion[channel]: conversion = db_retrieve_table_daemon( Conversion, unique_id=self.channels_measurement[channel].conversion_id) if conversion: meas = parse_measurement( self.channels_conversion[channel], self.channels_measurement[channel], measurements, channel, measurements[channel]) measurements[channel]['measurement'] = meas[channel][ 'measurement'] measurements[channel]['unit'] = meas[channel]['unit'] measurements[channel]['value'] = meas[channel]['value'] if measurements: self.logger.debug( "Adding measurements to influxdb: {}".format(measurements)) add_measurements_influxdb(self.unique_id, measurements) else: self.logger.debug("No measurements to add to influxdb.")
def download_data(self): self.logger.debug("Downloading Data") # Clear data previously stored in dictionary self.gadget.loggedDataReadout = {'Temp': {}, 'Humi': {}} # Download stored data starting from self.gadget.newestTimeStampMs self.gadget.readLoggedDataInterval( startMs=self.gadget.newestTimeStampMs) while self.running: if (not self.gadget.waitForNotifications(5) or not self.gadget.isLogReadoutInProgress()): break # Done reading data self.logger.debug("Downloaded Data") self.logger.debug("Parsing/saving data") list_timestamps_temp = [] list_timestamps_humi = [] # Store logged temperature self.logger.debug("Storing {} temperatures".format(len(self.gadget.loggedDataReadout['Temp']))) for each_ts, each_measure in self.gadget.loggedDataReadout['Temp'].items(): if not self.running: break if -40 > each_measure or each_measure > 125: continue # Temperature outside acceptable range list_timestamps_temp.append(each_ts) if self.is_enabled(0): datetime_ts = datetime.datetime.utcfromtimestamp(each_ts / 1000) measurement_single = { 0: { 'measurement': 'temperature', 'unit': 'C', 'value': each_measure } } measurement_single = parse_measurement( self.channels_conversion[0], self.channels_measurement[0], measurement_single, self.channels_measurement[0].channel, measurement_single[0]) write_influxdb_value( self.unique_id, measurement_single[0]['unit'], value=measurement_single[0]['value'], measure=measurement_single[0]['measurement'], channel=0, timestamp=datetime_ts) # Store logged humidity self.logger.debug("Storing {} humidities".format(len(self.gadget.loggedDataReadout['Humi']))) for each_ts, each_measure in self.gadget.loggedDataReadout['Humi'].items(): if not self.running: break if 0 >= each_measure or each_measure > 100: continue # Humidity outside acceptable range list_timestamps_humi.append(each_ts) if self.is_enabled(1): datetime_ts = datetime.datetime.utcfromtimestamp(each_ts / 1000) measurement_single = { 1: { 'measurement': 'humidity', 'unit': 'percent', 'value': each_measure } } measurement_single = parse_measurement( self.channels_conversion[1], self.channels_measurement[1], measurement_single, self.channels_measurement[1].channel, measurement_single[1]) write_influxdb_value( self.unique_id, measurement_single[1]['unit'], value=measurement_single[1]['value'], measure=measurement_single[1]['measurement'], channel=1, timestamp=datetime_ts) # Find common timestamps from both temperature and humidity lists list_timestamps_both = list(set(list_timestamps_temp).intersection(list_timestamps_humi)) self.logger.debug("Calculating/storing {} dewpoint and vpd".format(len(list_timestamps_both))) for each_ts in list_timestamps_both: if not self.running: break temperature = self.gadget.loggedDataReadout['Temp'][each_ts] humidity = self.gadget.loggedDataReadout['Humi'][each_ts] if (-200 > temperature or temperature > 200) or (0 > humidity or humidity > 100): continue # Measurement outside acceptable range datetime_ts = datetime.datetime.utcfromtimestamp(each_ts / 1000) # Calculate and store dew point if self.is_enabled(3) and self.is_enabled(0) and self.is_enabled(1): dewpoint = calculate_dewpoint(temperature, humidity) measurement_single = { 3: { 'measurement': 'dewpoint', 'unit': 'C', 'value': dewpoint } } measurement_single = parse_measurement( self.channels_conversion[3], self.channels_measurement[3], measurement_single, self.channels_measurement[3].channel, measurement_single[3]) write_influxdb_value( self.unique_id, measurement_single[3]['unit'], value=measurement_single[3]['value'], measure=measurement_single[3]['measurement'], channel=3, timestamp=datetime_ts) # Calculate and store vapor pressure deficit if self.is_enabled(4) and self.is_enabled(0) and self.is_enabled(1): vpd = calculate_vapor_pressure_deficit(temperature, humidity) measurement_single = { 4: { 'measurement': 'vapor_pressure_deficit', 'unit': 'Pa', 'value': vpd } } measurement_single = parse_measurement( self.channels_conversion[4], self.channels_measurement[4], measurement_single, self.channels_measurement[4].channel, measurement_single[4]) write_influxdb_value( self.unique_id, measurement_single[4]['unit'], value=measurement_single[4]['value'], measure=measurement_single[4]['measurement'], channel=4, timestamp=datetime_ts) # Download successfully finished, set newest timestamp self.gadget.newestTimeStampMs = self.gadget.tmp_newestTimeStampMs self.logger.debug("Parsed/saved data")
def get_new_data(self, past_seconds): # Basic implementation. Future development may use more complex library to access API endpoint = "https://{app}.data.thethingsnetwork.org/api/v2/query/{dev}?last={time}".format( app=self.application_id, dev=self.device_id, time="{}s".format(int(past_seconds))) headers = {"Authorization": "key {k}".format(k=self.app_api_key)} timestamp_format = '%Y-%m-%dT%H:%M:%S.%f' response = requests.get(endpoint, headers=headers) try: response.json() except ValueError: # No data returned self.logger.debug( "Response Error. Response: {}. Likely there is no data to be retrieved on TTN" .format(response.content)) return for each_resp in response.json(): if not self.running: break try: datetime_utc = datetime.datetime.strptime( each_resp['time'][:-7], timestamp_format) except Exception: # Sometimes the original timestamp is in milliseconds # instead of nanoseconds. Therefore, remove 3 less digits # past the decimal and try again to parse. try: datetime_utc = datetime.datetime.strptime( each_resp['time'][:-4], timestamp_format) except Exception as e: self.logger.error( "Could not parse timestamp '{}': {}".format( each_resp['time'], e)) continue # Malformed timestamp encountered. Discard measurement. if (not self.latest_datetime or self.latest_datetime < datetime_utc): self.latest_datetime = datetime_utc measurements = {} for channel in self.channels_measurement: if (self.is_enabled(channel) and self.options_channels['variable_name'][channel] in each_resp and each_resp[self.options_channels['variable_name'] [channel]] is not None): # Original value/unit measurements[channel] = {} measurements[channel][ 'measurement'] = self.channels_measurement[ channel].measurement measurements[channel]['unit'] = self.channels_measurement[ channel].unit measurements[channel]['value'] = each_resp[ self.options_channels['variable_name'][channel]] measurements[channel]['timestamp_utc'] = datetime_utc # Convert value/unit is conversion_id present and valid if self.channels_conversion[channel]: conversion = db_retrieve_table_daemon( Conversion, unique_id=self.channels_measurement[channel]. conversion_id) if conversion: meas = parse_measurement( self.channels_conversion[channel], self.channels_measurement[channel], measurements, channel, measurements[channel], timestamp=datetime_utc) measurements[channel]['measurement'] = meas[ channel]['measurement'] measurements[channel]['unit'] = meas[channel][ 'unit'] measurements[channel]['value'] = meas[channel][ 'value'] if measurements: self.logger.debug( "Adding measurements to influxdb: {}".format(measurements)) add_measurements_influxdb( self.unique_id, measurements, use_same_timestamp=INPUT_INFORMATION[ 'measurements_use_same_timestamp']) else: self.logger.debug("No measurements to add to influxdb.") # set datetime to latest timestamp if self.running: with session_scope(MYCODO_DB_PATH) as new_session: mod_input = new_session.query(Input).filter( Input.unique_id == self.unique_id).first() if not mod_input.datetime or mod_input.datetime < self.latest_datetime: mod_input.datetime = self.latest_datetime new_session.commit()
def get_measurement(self): """Gets the temperature and humidity.""" # # Initialize measurements dictionary # measurements = {} for channel in self.channels_measurement: if self.is_enabled(channel): # Initialize channel dictionary measurements[channel] = {} # # Set the measurement and unit # measurements[channel][ 'measurement'] = self.channels_measurement[ channel].measurement measurements[channel]['unit'] = self.channels_measurement[ channel].unit # # Set the measurement value # measurements[channel]['value'] = self.random.randint(50, 70) self.logger.info( "Channel {} is enabled and storing a value of {} " "with measurement {} and unit {}".format( channel, measurements[channel]['value'], measurements[channel]['measurement'], measurements[channel]['unit'])) # Convert value/unit is conversion_id present and valid if self.channels_conversion[channel]: conversion = db_retrieve_table_daemon( Conversion, unique_id=self.channels_measurement[channel]. conversion_id) if conversion: meas = parse_measurement( self.channels_conversion[channel], self.channels_measurement[channel], measurements, channel, measurements[channel]) measurements[channel]['measurement'] = meas[channel][ 'measurement'] measurements[channel]['unit'] = meas[channel]['unit'] measurements[channel]['value'] = meas[channel]['value'] if measurements: self.logger.debug( "Adding measurements to influxdb: {}".format(measurements)) add_measurements_influxdb(self.unique_id, measurements, use_same_timestamp=INPUT_INFORMATION[ 'measurements_use_same_timestamp']) else: self.logger.debug("No measurements to add to influxdb.") self.logger.info("This INFO message will always be displayed. " "self.fan_modulate: {}, " "self.fan_seconds: {}, " "self.measure_range: {}.".format( self.fan_modulate, self.fan_seconds, self.measure_range)) self.logger.debug("This DEBUG message will only be displayed if the " "Debug option is enabled.") return self.return_dict
def get_new_data(self, past_seconds): try: seconds = int(past_seconds) except: self.logger.error("past_seconds does not represent an integer") return endpoint = "https://nam1.cloud.thethings.network" \ "/api/v3/as/applications/{app}/devices/{dev}/packages/storage/uplink_message?" \ "last={time}s&field_mask=up.uplink_message.decoded_payload".format( app=self.application_id, dev=self.device_id, time=seconds) headers = {"Authorization": "Bearer {k}".format(k=self.app_api_key)} self.logger.debug("endpoint: {}".format(endpoint)) self.logger.debug("headers: {}".format(headers)) # Get measurement data from TTN try: response = requests.get(endpoint, headers=headers) except requests.exceptions.ConnectionError as err: self.logger.error( "requests.exceptions.ConnectionError: {}".format(err)) return except Exception as err: self.logger.error("Exception: {}".format(err)) return if response.status_code != 200: self.logger.info("response.status_code != 200: {}".format( response.reason)) self.logger.debug("response.content: {}".format(response.content)) list_dicts = response.content.decode().split("\n") self.logger.debug("list_dicts: {}".format(list_dicts)) for each_resp in list_dicts: measurements = {} if not self.running: break if not each_resp: continue self.logger.debug("each_resp: {}".format(each_resp)) try: resp_json = json.loads(each_resp) except: resp_json = {} self.logger.debug("resp_json: {}".format(resp_json)) try: datetime_utc = datetime.datetime.strptime( resp_json['result']['received_at'][:-7], self.timestamp_format) except Exception: # Sometimes the original timestamp is in milliseconds # instead of nanoseconds. Therefore, remove 3 less digits # past the decimal and try again to parse. try: datetime_utc = datetime.datetime.strptime( resp_json['result']['received_at'][:-4], self.timestamp_format) except Exception as e: self.logger.error( "Could not parse timestamp '{}': {}".format( resp_json['result']['received_at'], e)) continue # Malformed timestamp encountered. Discard measurement. if ('result' not in resp_json or 'uplink_message' not in resp_json['result'] or 'decoded_payload' not in resp_json['result']['uplink_message'] or not resp_json['result']['uplink_message'] ['decoded_payload']): self.logger.debug( "resp_json empty or malformed: {}".format(resp_json)) continue payload = resp_json['result']['uplink_message']['decoded_payload'] if (not self.latest_datetime or self.latest_datetime < datetime_utc): self.latest_datetime = datetime_utc for channel in self.channels_measurement: jmespath_expression = self.options_channels[ 'jmespath_expression'][channel] try: jmesexpression = self.jmespath.compile(jmespath_expression) value = jmesexpression.search(payload) self.logger.debug("Expression: {}, value: {}".format( jmespath_expression, value)) except Exception as err: self.logger.error( "Error in JSON '{}' finding expression '{}': {}". format(payload, jmespath_expression, err)) continue if value is None: continue # Original value/unit measurements[channel] = {} measurements[channel][ 'measurement'] = self.channels_measurement[ channel].measurement measurements[channel]['unit'] = self.channels_measurement[ channel].unit measurements[channel]['value'] = value measurements[channel]['timestamp_utc'] = datetime_utc # Convert value/unit is conversion_id present and valid if self.channels_conversion[channel]: conversion = db_retrieve_table_daemon( Conversion, unique_id=self.channels_measurement[channel]. conversion_id) if conversion: meas = parse_measurement( self.channels_conversion[channel], self.channels_measurement[channel], measurements, channel, measurements[channel], timestamp=datetime_utc) measurements[channel]['measurement'] = meas[channel][ 'measurement'] measurements[channel]['unit'] = meas[channel]['unit'] measurements[channel]['value'] = meas[channel]['value'] if measurements: self.logger.debug( "Adding measurements to influxdb: {}".format(measurements)) add_measurements_influxdb( self.unique_id, measurements, use_same_timestamp=INPUT_INFORMATION[ 'measurements_use_same_timestamp']) else: self.logger.debug("No measurements to add to influxdb.") # set datetime to latest timestamp if self.running: with session_scope(MYCODO_DB_PATH) as new_session: mod_input = new_session.query(Input).filter( Input.unique_id == self.unique_id).first() if not mod_input.datetime or mod_input.datetime < self.latest_datetime: mod_input.datetime = self.latest_datetime new_session.commit()