def data_get_current(module_id, group_id, sensor_id):
data = []
sensor = utils.get_sensor(module_id, group_id, sensor_id)
if sensor is None:
log.error("[" + module_id + "][" + group_id + "][" + sensor_id +
"] sensor not found")
return json.dumps(data)
if "plugin" in sensor and "poll_on_demand" in sensor["plugin"] and sensor[
"plugin"]["poll_on_demand"]:
# the sensor needs to be polled on demand
run(module_id, group_id, sensor_id, "save")
key = conf["constants"]["db_schema"][
"root"] + ":" + module_id + ":" + group_id + ":" + sensor_id
# return the latest measure
data = db.range(
key,
withscores=False,
milliseconds=True,
formatter=conf["constants"]["formats"][sensor["format"]]["formatter"])
# if an image, decode it and return it
if sensor["format"] == "image":
return base64.b64decode(data[0])
# if a calendar, return the current value
elif sensor["format"] == "calendar":
return json.dumps(utils.parse_calendar(data))
else:
return json.dumps(data)
def parse(sensor):
# retrieve the raw data from the cache
data = db.range(sensor["db_cache"], withscores=False)
if len(data) == 0:
log.warning("[" + sensor["module_id"] + "][" + sensor["group_id"] +
"][" + sensor["sensor_id"] + "]: no data to parse")
return None
data = data[0]
measures = None
try:
# parse the cached data
measures = plugins[sensor['plugin']['plugin_name']].parse(sensor, data)
if not isinstance(measures, list):
# returned a single value, build the data structure
value = measures
measures = []
measure = {}
measure["key"] = sensor["sensor_id"]
measure["value"] = value
measures.append(measure)
# format each value
for i in range(len(measures)):
# post-process the measure if configured
if "command_transform" in sensor:
orig_value = measures[i]["value"]
command = sensor["command_transform"].replace(
"%value%", str(orig_value))
measures[i]["value"] = utils.run_command(command)
log.debug("[" + sensor["module_id"] + "][" +
sensor["group_id"] + "][" + sensor["sensor_id"] +
"] transforming " + str(orig_value) + " into " +
str(measures[i]["value"]))
# normalize the measures
if sensor["format"] == "temperature":
measures[i]["value"] = utils.temperature_unit(
measures[i]["value"])
if sensor["format"] == "length":
measures[i]["value"] = utils.length_unit(measures[i]["value"])
if sensor["format"] == "pressure":
measures[i]["value"] = utils.pressure_unit(
measures[i]["value"])
if sensor["format"] == "speed":
measures[i]["value"] = utils.speed_unit(measures[i]["value"])
measures[i]["value"] = utils.normalize(
measures[i]["value"],
conf["constants"]["formats"][sensor["format"]]["formatter"])
log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] +
"][" + sensor["sensor_id"] + "] parsed: " + str(measures))
except Exception, e:
log.warning("[" + sensor["module_id"] + "][" + sensor["group_id"] +
"][" + sensor["sensor_id"] + "] unable to parse " +
str(data) + ": " + utils.get_exception(e))
return None
def run():
url = "https://weatherstation.wunderground.com/weatherstation/updateweatherstation.php"
measures = {}
# prepare the custom parameters
count = 0
for field, sensor_key in conf["pws"]["data"].iteritems():
# for each mapping, retrieve the sensor
sensor = utils.get_sensor_string(sensor_key)
if sensor is None:
log.warning("invalid sensor " + sensor_key +
" associated to field " + field)
continue
# retrieve the data
key = conf["constants"]["db_schema"]["root"] + ":" + sensor_key
data = db.range(key, withscores=True)
if len(data) == 0: continue
timestamp = data[0][0]
value = data[0][1]
# do not send measure already sent
if new_measures_only and timestamp < utils.now(
) - conf["pws"]["publishing_interval"] * 60:
continue
# perform the appropriate conversion
if field in [
"windspeedmph", "windgustmph", "windspdmph_avg2m",
"windgustmph_10m"
] and not conf["general"]["units"]["imperial"]:
value = utils.speed_unit(value, force=True)
if field in ["dewptf", "tempf", "soiltempf"
] and not conf["general"]["units"]["fahrenheit"]:
value = utils.temperature_unit(value, force=True)
if field in ["rainin", "dailyrainin", "baromin"
] and not conf["general"]["units"]["imperial"]:
value = utils.pressure_unit(value, force=True)
measures[field] = value
count = count + 1
# if at least one parameter needs to be updated
if count > 0:
log.debug("Prepare uploading to pws " + str(measures))
# prepare the common parameter
params = {}
params["action"] = "updateraw"
params["ID"] = conf["pws"]["username"]
params["PASSWORD"] = conf["pws"]["password"]
params["dateutc"] = "now"
params.update(measures)
response = utils.web_get(url, params=params)
if "success" in response:
log.info("Updated the PWS " + conf["pws"]["username"] + " with: " +
str(measures))
else:
log.error("failed to update the PWS: " + str(response))
def data_get_current_timestamp(module_id, group_id, sensor_id):
data = []
sensor = utils.get_sensor(module_id, group_id, sensor_id)
if sensor is None:
log.error("[" + module_id + "][" + group_id + "][" + sensor_id +
"] sensor not found")
return json.dumps(data)
key = conf["constants"]["db_schema"][
"root"] + ":" + module_id + ":" + group_id + ":" + sensor_id
data = db.range(key, withscores=True, milliseconds=True)
if len(data) > 0:
return json.dumps(
[utils.timestamp_difference(utils.now(), data[0][0] / 1000)])
else:
return json.dumps(data)
def data_get_calendar(module_id, group_id, sensor_id):
data = []
sensor = utils.get_sensor(module_id, group_id, sensor_id)
if sensor is None:
log.error("[" + module_id + "][" + group_id + "][" + sensor_id +
"] sensor not found")
return json.dumps(data)
key = conf["constants"]["db_schema"][
"root"] + ":" + module_id + ":" + group_id + ":" + sensor_id
# return the latest measure
data = db.range(
key,
withscores=False,
milliseconds=True,
formatter=conf["constants"]["formats"][sensor["format"]]["formatter"])
return json.dumps(data)
def save(sensor, force=False):
cache_timestamp = 0
# get the raw data from the cache
if db.exists(sensor["db_cache"]):
data = db.range(sensor["db_cache"], withscores=True)
cache_timestamp = data[0][0]
# if too old, refresh it
cache_expire_min = sensor["plugin"][
"cache_expire_min"] if "plugin" in sensor and "cache_expire_min" in sensor[
"plugin"] else conf["sensors"]["cache_expire_min"]
if force or (utils.now() - cache_timestamp
) > cache_expire_min * conf["constants"]["1_minute"]:
# if an exception occurred, skip this sensor
if poll(sensor) is None: return
# get the parsed data
measures = parse(sensor)
# store it into the database
store(sensor, measures)
def store(sensor, measures, ifnotexists=False):
# if an exception occurred, skip this sensor
if measures is None: return
# for each returned measure
for measure in measures:
# set the timestamp to now if not already set
if "timestamp" not in measure: measure["timestamp"] = utils.now()
# define the key to store the value
key = sensor["db_group"] + ":" + measure["key"]
# if ifnotexists is set, check if the key exists
if ifnotexists and db.exists(key):
log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] +
"][" + sensor["sensor_id"] +
"] key already exists, ignoring new value")
return
# delete previous values if needed
realtime_count = conf["sensors"]["retention"]["realtime_count"]
if "retention" in sensor and "realtime_count" in sensor["retention"]:
realtime_count = sensor["retention"]["realtime_count"]
if realtime_count > 0:
db.deletebyrank(key, 0, -realtime_count)
# if only measures with a newer timestamp than the latest can be added, apply the policy
realtime_new_only = conf["sensors"]["retention"]["realtime_new_only"]
if "retention" in sensor and "realtime_new_only" in sensor["retention"]:
realtime_new_only = sensor["retention"]["realtime_new_only"]
if realtime_new_only:
# retrieve the latest measure's timestamp
last = db.range(key, -1, -1)
if len(last) > 0:
last_timestamp = last[0][0]
# if the measure's timestamp is older or the same, skip it
if measure["timestamp"] <= last_timestamp:
log.debug("[" + sensor["module_id"] + "][" +
sensor["group_id"] + "][" + sensor["sensor_id"] +
"] (" +
utils.timestamp2date(measure["timestamp"]) +
") old event, ignoring " + measure["key"] +
": " + str(measure["value"]))
continue
# check if there is already something stored with the same timestamp
old = db.rangebyscore(key, measure["timestamp"], measure["timestamp"])
if len(old) > 0:
if old[0][1] == measure["value"]:
# if the value is also the same, skip it
log.debug("[" + sensor["module_id"] + "][" +
sensor["group_id"] + "][" + sensor["sensor_id"] +
"] (" + utils.timestamp2date(measure["timestamp"]) +
") already in the database, ignoring " +
measure["key"] + ": " + str(measure["value"]))
continue
else:
# same timestamp but different value, remove the old value so to store the new one
db.deletebyscore(key, measure["timestamp"],
measure["timestamp"])
# store the value into the database
log.info("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" +
sensor["sensor_id"] + "] (" +
utils.timestamp2date(measure["timestamp"]) + ") saving " +
measure["key"] + ": " +
utils.truncate(str(measure["value"])) +
conf["constants"]["formats"][sensor["format"]]["suffix"])
db.set(key, measure["value"], measure["timestamp"])
# re-calculate the derived measures for the hour/day
if "summarize" in sensor:
summarize(sensor, 'hour', utils.hour_start(measure["timestamp"]),
utils.hour_end(measure["timestamp"]))
summarize(sensor, 'day', utils.day_start(measure["timestamp"]),
utils.day_end(measure["timestamp"]))