def data_get_current(module_id, group_id, sensor_id): data = [] sensor = utils.get_sensor(module_id, group_id, sensor_id) if sensor is None: log.error("[" + module_id + "][" + group_id + "][" + sensor_id + "] sensor not found") return json.dumps(data) if "plugin" in sensor and "poll_on_demand" in sensor["plugin"] and sensor[ "plugin"]["poll_on_demand"]: # the sensor needs to be polled on demand run(module_id, group_id, sensor_id, "save") key = conf["constants"]["db_schema"][ "root"] + ":" + module_id + ":" + group_id + ":" + sensor_id # return the latest measure data = db.range( key, withscores=False, milliseconds=True, formatter=conf["constants"]["formats"][sensor["format"]]["formatter"]) # if an image, decode it and return it if sensor["format"] == "image": return base64.b64decode(data[0]) # if a calendar, return the current value elif sensor["format"] == "calendar": return json.dumps(utils.parse_calendar(data)) else: return json.dumps(data)
def parse(sensor): # retrieve the raw data from the cache data = db.range(sensor["db_cache"], withscores=False) if len(data) == 0: log.warning("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "]: no data to parse") return None data = data[0] measures = None try: # parse the cached data measures = plugins[sensor['plugin']['plugin_name']].parse(sensor, data) if not isinstance(measures, list): # returned a single value, build the data structure value = measures measures = [] measure = {} measure["key"] = sensor["sensor_id"] measure["value"] = value measures.append(measure) # format each value for i in range(len(measures)): # post-process the measure if configured if "command_transform" in sensor: orig_value = measures[i]["value"] command = sensor["command_transform"].replace( "%value%", str(orig_value)) measures[i]["value"] = utils.run_command(command) log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] transforming " + str(orig_value) + " into " + str(measures[i]["value"])) # normalize the measures if sensor["format"] == "temperature": measures[i]["value"] = utils.temperature_unit( measures[i]["value"]) if sensor["format"] == "length": measures[i]["value"] = utils.length_unit(measures[i]["value"]) if sensor["format"] == "pressure": measures[i]["value"] = utils.pressure_unit( measures[i]["value"]) if sensor["format"] == "speed": measures[i]["value"] = utils.speed_unit(measures[i]["value"]) measures[i]["value"] = utils.normalize( measures[i]["value"], conf["constants"]["formats"][sensor["format"]]["formatter"]) log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] parsed: " + str(measures)) except Exception, e: log.warning("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] unable to parse " + str(data) + ": " + utils.get_exception(e)) return None
def run(): url = "https://weatherstation.wunderground.com/weatherstation/updateweatherstation.php" measures = {} # prepare the custom parameters count = 0 for field, sensor_key in conf["pws"]["data"].iteritems(): # for each mapping, retrieve the sensor sensor = utils.get_sensor_string(sensor_key) if sensor is None: log.warning("invalid sensor " + sensor_key + " associated to field " + field) continue # retrieve the data key = conf["constants"]["db_schema"]["root"] + ":" + sensor_key data = db.range(key, withscores=True) if len(data) == 0: continue timestamp = data[0][0] value = data[0][1] # do not send measure already sent if new_measures_only and timestamp < utils.now( ) - conf["pws"]["publishing_interval"] * 60: continue # perform the appropriate conversion if field in [ "windspeedmph", "windgustmph", "windspdmph_avg2m", "windgustmph_10m" ] and not conf["general"]["units"]["imperial"]: value = utils.speed_unit(value, force=True) if field in ["dewptf", "tempf", "soiltempf" ] and not conf["general"]["units"]["fahrenheit"]: value = utils.temperature_unit(value, force=True) if field in ["rainin", "dailyrainin", "baromin" ] and not conf["general"]["units"]["imperial"]: value = utils.pressure_unit(value, force=True) measures[field] = value count = count + 1 # if at least one parameter needs to be updated if count > 0: log.debug("Prepare uploading to pws " + str(measures)) # prepare the common parameter params = {} params["action"] = "updateraw" params["ID"] = conf["pws"]["username"] params["PASSWORD"] = conf["pws"]["password"] params["dateutc"] = "now" params.update(measures) response = utils.web_get(url, params=params) if "success" in response: log.info("Updated the PWS " + conf["pws"]["username"] + " with: " + str(measures)) else: log.error("failed to update the PWS: " + str(response))
def data_get_current_timestamp(module_id, group_id, sensor_id): data = [] sensor = utils.get_sensor(module_id, group_id, sensor_id) if sensor is None: log.error("[" + module_id + "][" + group_id + "][" + sensor_id + "] sensor not found") return json.dumps(data) key = conf["constants"]["db_schema"][ "root"] + ":" + module_id + ":" + group_id + ":" + sensor_id data = db.range(key, withscores=True, milliseconds=True) if len(data) > 0: return json.dumps( [utils.timestamp_difference(utils.now(), data[0][0] / 1000)]) else: return json.dumps(data)
def data_get_calendar(module_id, group_id, sensor_id): data = [] sensor = utils.get_sensor(module_id, group_id, sensor_id) if sensor is None: log.error("[" + module_id + "][" + group_id + "][" + sensor_id + "] sensor not found") return json.dumps(data) key = conf["constants"]["db_schema"][ "root"] + ":" + module_id + ":" + group_id + ":" + sensor_id # return the latest measure data = db.range( key, withscores=False, milliseconds=True, formatter=conf["constants"]["formats"][sensor["format"]]["formatter"]) return json.dumps(data)
def save(sensor, force=False): cache_timestamp = 0 # get the raw data from the cache if db.exists(sensor["db_cache"]): data = db.range(sensor["db_cache"], withscores=True) cache_timestamp = data[0][0] # if too old, refresh it cache_expire_min = sensor["plugin"][ "cache_expire_min"] if "plugin" in sensor and "cache_expire_min" in sensor[ "plugin"] else conf["sensors"]["cache_expire_min"] if force or (utils.now() - cache_timestamp ) > cache_expire_min * conf["constants"]["1_minute"]: # if an exception occurred, skip this sensor if poll(sensor) is None: return # get the parsed data measures = parse(sensor) # store it into the database store(sensor, measures)
def store(sensor, measures, ifnotexists=False): # if an exception occurred, skip this sensor if measures is None: return # for each returned measure for measure in measures: # set the timestamp to now if not already set if "timestamp" not in measure: measure["timestamp"] = utils.now() # define the key to store the value key = sensor["db_group"] + ":" + measure["key"] # if ifnotexists is set, check if the key exists if ifnotexists and db.exists(key): log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] key already exists, ignoring new value") return # delete previous values if needed realtime_count = conf["sensors"]["retention"]["realtime_count"] if "retention" in sensor and "realtime_count" in sensor["retention"]: realtime_count = sensor["retention"]["realtime_count"] if realtime_count > 0: db.deletebyrank(key, 0, -realtime_count) # if only measures with a newer timestamp than the latest can be added, apply the policy realtime_new_only = conf["sensors"]["retention"]["realtime_new_only"] if "retention" in sensor and "realtime_new_only" in sensor["retention"]: realtime_new_only = sensor["retention"]["realtime_new_only"] if realtime_new_only: # retrieve the latest measure's timestamp last = db.range(key, -1, -1) if len(last) > 0: last_timestamp = last[0][0] # if the measure's timestamp is older or the same, skip it if measure["timestamp"] <= last_timestamp: log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] (" + utils.timestamp2date(measure["timestamp"]) + ") old event, ignoring " + measure["key"] + ": " + str(measure["value"])) continue # check if there is already something stored with the same timestamp old = db.rangebyscore(key, measure["timestamp"], measure["timestamp"]) if len(old) > 0: if old[0][1] == measure["value"]: # if the value is also the same, skip it log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] (" + utils.timestamp2date(measure["timestamp"]) + ") already in the database, ignoring " + measure["key"] + ": " + str(measure["value"])) continue else: # same timestamp but different value, remove the old value so to store the new one db.deletebyscore(key, measure["timestamp"], measure["timestamp"]) # store the value into the database log.info("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] (" + utils.timestamp2date(measure["timestamp"]) + ") saving " + measure["key"] + ": " + utils.truncate(str(measure["value"])) + conf["constants"]["formats"][sensor["format"]]["suffix"]) db.set(key, measure["value"], measure["timestamp"]) # re-calculate the derived measures for the hour/day if "summarize" in sensor: summarize(sensor, 'hour', utils.hour_start(measure["timestamp"]), utils.hour_end(measure["timestamp"])) summarize(sensor, 'day', utils.day_start(measure["timestamp"]), utils.day_end(measure["timestamp"]))