def run():
if not plugin_conf["enabled"]: return
# kill rtl_433 if running
utils.run_command("killall rtl_433")
# run rtl_433 and handle the output
command = plugin_conf['command'] + " " + command_arguments
log.debug("[" + __name__ + "] running command " + command)
process = subprocess.Popen(shlex.split(command),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
prev_output = ""
while True:
# read a line from the output
output = process.stdout.readline()
if output == '' and process.poll() is not None:
# process ended, break
log.info("[" + __name__ + "] rtl_433 has ended")
break
if output:
# output available
try:
# avoid handling the same exact output, skipping
if prev_output == output: continue
# parse the json output
json_output = json.loads(output)
except ValueError, e:
# not a valid json, ignoring
continue
# for each registered search string
for search_string in nodes:
# check if the output matches the search string
search_json = json.loads(search_string)
found = True
for key, value in search_json.iteritems():
# check every key/value pair
if key not in json_output: found = False
if str(value) != str(json_output[key]): found = False
if not found: continue
# found, save each measure
node = nodes[search_string]
measures = []
for measure in node:
sensor = node[measure]
# create the measure data structure
measure_data = {}
if "time" in json_output:
date = datetime.datetime.strptime(
json_output["time"], "%Y-%m-%d %H:%M:%S")
measure_data["timestamp"] = utils.timezone(
utils.timezone(int(time.mktime(date.timetuple()))))
measure_data["key"] = sensor["sensor_id"]
value = json_output[
measure] if measure in json_output else default_value
measure_data["value"] = utils.normalize(
value, conf["constants"]["formats"][sensor["format"]]
["formatter"])
measures.append(measure_data)
sensors.store(sensor, measures)
# keep track of the last line of output
prev_output = output
def parse(sensor,data):
measures = []
measure = {}
# load the file
data = json.loads(data)
# for each line
for line in data:
entry = line.split(',')
measure = {}
# if a filter is defined, ignore the line if the filter is not found
if "filter" in sensor["plugin"] and entry[sensor["plugin"]["filter_position"+1]] != sensor["plugin"]["filter"]: continue
# if a prefix is defined, filter based on it
if "prefix" in sensor["plugin"] and not entry[sensor["plugin"]["value_position"+1]].startswith(sensor['plugin']['prefix']): continue
# generate the timestamp
if "date_position" in sensor["plugin"]:
date = datetime.datetime.strptime(entry[sensor["plugin"]["date_position"+1]],sensor["plugin"]["date_format"])
measure["timestamp"] = utils.timezone(utils.timezone(int(time.mktime(date.timetuple()))))
else: measure["timestamp"] = utils.now()
# set the key as the sensor_id
measure["key"] = sensor["sensor_id"]
# strip out the measure from the value
value = entry[sensor["plugin"]["value_position"+1]]
# if a measure prefix was defined, remove it
if "prefix" in sensor["plugin"]: value.replace(sensor['plugin']['prefix'],"")
# set the value
measure["value"] = utils.normalize(value,conf["constants"]["formats"][sensor["format"]]["formatter"])
measures.append(measure)
return measures
def parse(sensor, data):
measures = []
measure = {}
# load the file
data = json.loads(data)
# for each line
for line in data.split('\n'):
#EventID|Time|Latitude|Longitude|Depth/Km|Author|Catalog|Contributor|ContributorID|MagType|Magnitude|MagAuthor|EventLocationName
# 0 1 2 3 4 5 6 7 8 9 10 11 12
if line.startswith('#'): continue
measure = {}
# split the entries
entry = line.split('|')
if len(entry) != 13: continue
# set the timestamp to the event's date
date_format = "%Y-%m-%dT%H:%M:%S.%f"
date = datetime.datetime.strptime(entry[1], date_format)
measure["timestamp"] = utils.timezone(
utils.timezone(int(time.mktime(date.timetuple()))))
# prepare the position value
position = {}
position["latitude"] = float(entry[2])
position["longitude"] = float(entry[3])
position["label"] = str(entry[10])
date_string = utils.timestamp2date(int(measure["timestamp"]))
# position["text"] = str("<p><b>"+entry[12]+":</b></p><p>Magnitude: "+entry[10]+"</p><p>Date: "+date_string+"</p><p>Depth: "+entry[4]+" km</p>")
position["text"] = str(entry[12])
# prepare the measure
measure["key"] = sensor["sensor_id"] + ":day:avg"
measure["value"] = json.dumps(position)
# add the event to the measures
measures.append(measure)
return measures
def parse(sensor, data):
data = json.loads(data)
device = {}
# for each device
for device_name in data:
# identify the device
if device_name != sensor["plugin"]["device_name"]: continue
# normalize the data for a map
date = utils.timestamp2date(
utils.timezone(int(data[device_name]["timeStamp"] / 1000)))
device["label"] = str(device_name)
device["text"] = str("<p><b>" + device_name + ":</b></p><p>" + date +
" (" + data[device_name]["positionType"] +
") </p>")
device["latitude"] = data[device_name]["latitude"]
device["longitude"] = data[device_name]["longitude"]
device["accuracy"] = data[device_name]["horizontalAccuracy"]
return json.dumps(device)
def upgrade_2_0():
######## START OF CONFIGURATION
# remote all data from the target database
empty_target_db = False
# migrate history data
migrate_history = True
# history start timestamp to migrate, "-inf" for all
history_start_timestamp = "-inf"
# historu end timestamp to migrate
history_end_timestamp = utils.now()
# migrate recent data
migrate_recent = True
# database number from which we are migrating
db_from = 1
# database number into which we are migrating
db_to = 2
# debug
debug = False
# keys to migrate history (from key -> to key)
# destination key format: myHouse:<module_id>:<group_id>:<sensor_id>
history = {
'home:weather:outdoor:temperature:day:max':
'myHouse:outdoor:temperature:external:day:max',
'home:weather:outdoor:temperature:day:min':
'myHouse:outdoor:temperature:external:day:min',
'home:weather:outdoor:temperature:day':
'myHouse:outdoor:temperature:external:day:avg',
'home:weather:indoor:temperature:day:max':
'myHouse:indoor:temperature:living_room:day:max',
'home:weather:indoor:temperature:day:min':
'myHouse:indoor:temperature:living_room:day:min',
'home:weather:indoor:temperature:day':
'myHouse:indoor:temperature:living_room:day:avg',
'home:weather:almanac:record:min':
'myHouse:outdoor:temperature:record:day:min',
'home:weather:almanac:record:max':
'myHouse:outdoor:temperature:record:day:max',
'home:weather:almanac:normal:min':
'myHouse:outdoor:temperature:normal:day:min',
'home:weather:almanac:normal:max':
'myHouse:outdoor:temperature:normal:day:max',
'home:weather:outdoor:condition:day':
'myHouse:outdoor:temperature:condition:day:avg',
}
# keys to migrate recent data (from key -> to key)
recent = {
'home:weather:outdoor:temperature:measure':
'myHouse:outdoor:temperature:external',
'home:weather:indoor:temperature:measure':
'myHouse:indoor:temperature:living_room',
'home:weather:outdoor:condition:measure':
'myHouse:outdoor:temperature:condition',
}
######## END OF CONFIGURATION
conf = config.get_config(validate=False)
print "[Migration from v1.x to v2.0]\n"
input(
"WARNING: which data will be migrate is defined within this script, on top of the upgrade_20() function.\nIndividual sensors to migrate must be specified manually\nPlase ensure you have reviewed all the settings first!\n\nPress Enter to continue..."
)
backup("1.0")
# empty the target database first
if empty_target_db:
print "Flushing target database..."
change_db(db_to)
db.flushdb()
# for each history key to migrate
print "Migrating historical data..."
for key_from in history:
if not migrate_history: break
key_to = history[key_from]
print "\tMigrating " + key_from + " -> " + key_to
# retrieve all the data
change_db(db_from)
data = db.rangebyscore(key_from,
history_start_timestamp,
history_end_timestamp,
withscores=True)
change_db(db_to)
count = 0
# for each entry
for entry in data:
timestamp = utils.day_start(utils.timezone(entry[0]))
value = utils.normalize(entry[1])
# store it into the new database
if debug:
print "[HISTORY][" + key_to + "] (" + utils.timestamp2date(
timestamp) + ") " + str(value)
db.set(key_to, value, timestamp)
count = count + 1
print "\t\tdone, " + str(count) + " values"
# for each recent key to migrate
print "Migrating recent data..."
for key_from in recent:
if not migrate_recent: break
key_to = recent[key_from]
print "\tMigrating " + key_from + " -> " + key_to
# retrieve the recent data
change_db(db_from)
data = db.rangebyscore(key_from,
utils.now() - 2 * conf["constants"]["1_day"],
utils.now(),
withscores=True)
change_db(db_to)
count = 0
# for each entry
for entry in data:
timestamp = utils.timezone(entry[0])
value = utils.normalize(entry[1])
if debug:
print "[RECENT][" + key_to + "] (" + utils.timestamp2date(
timestamp) + ") " + str(value)
# skip it if the same value is already stored
old = db.rangebyscore(key_to, timestamp, timestamp)
if len(old) > 0: continue
# store it into the new database
db.set(key_to, value, timestamp)
# create the sensor data structure
key_split = key_to.split(":")
group_id = key_split[-2]
sensor_id = key_split[-1]
module_id = key_split[-4]
sensor = utils.get_sensor(module_id, group_id, sensor_id)
sensor['module_id'] = module_id
sensor['group_id'] = group_id
sensor['db_group'] = conf["constants"]["db_schema"][
"root"] + ":" + sensor["module_id"] + ":" + sensor["group_id"]
sensor[
'db_sensor'] = sensor['db_group'] + ":" + sensor["sensor_id"]
import sensors
sensors.summarize(sensor, 'hour', utils.hour_start(timestamp),
utils.hour_end(timestamp))
count = count + 1
print "\t\tdone, " + str(count) + " values"
print "Upgrading database..."
version_key = conf["constants"]["db_schema"]["version"]
db.set_simple(version_key, "2.0")
def parse(sensor, data):
request = sensor['plugin']['measure']
measures = []
measure = {}
measure["key"] = sensor["sensor_id"]
# parse the json
parsed_json = json.loads(data)
if "error" in parsed_json:
# error returned
log.error("[" + sensor["module_id"] + "][" + sensor["group_id"] +
"][" + sensor["sensor_id"] + "] " +
parsed_json["error"]["type"] + ": " +
parsed_json["error"]["description"])
return None
if request == "temperature":
measure["value"] = float(parsed_json['current_observation']['temp_c'])
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "humidity":
measure["value"] = int(
parsed_json['current_observation']['relative_humidity'].replace(
'%', ''))
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "wind":
measure["value"] = float(
parsed_json['current_observation']['wind_kph'])
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "wind_gust":
measure["value"] = float(
parsed_json['current_observation']['wind_gust_kph'])
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "pressure":
measure["value"] = float(
parsed_json['current_observation']['pressure_mb'])
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "condition":
measure["value"] = parsed_json['current_observation']['icon']
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "wind_dir":
direction = parsed_json['current_observation']['wind_dir']
if len(direction) > 0 and (direction[0] == "N" or direction[0] == "W"
or direction[0] == "S"
or direction[0] == "E"):
direction = direction[0]
else:
direction = "-"
measure["value"] = direction
measure["timestamp"] = utils.timezone(
int(parsed_json['current_observation']['observation_epoch']))
measures.append(measure)
elif request == "forecast_condition":
for entry in parsed_json['forecast']['simpleforecast'][
'forecastday'][:forecast_max_entries]:
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:avg"
measure["timestamp"] = utils.day_start(
utils.timezone(int(entry["date"]["epoch"])))
measure["value"] = entry["icon"]
measures.append(measure)
elif request == "forecast_pop":
for entry in parsed_json['forecast']['simpleforecast'][
'forecastday'][:forecast_max_entries]:
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:avg"
measure["timestamp"] = utils.day_start(
utils.timezone(int(entry["date"]["epoch"])))
measure["value"] = entry["pop"] if entry["pop"] > 0 else 0
measures.append(measure)
elif request == "forecast_rain":
for entry in parsed_json['forecast']['simpleforecast'][
'forecastday'][:forecast_max_entries]:
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:avg"
measure["timestamp"] = utils.day_start(
utils.timezone(int(entry["date"]["epoch"])))
measure["value"] = entry["qpf_allday"][
"mm"] if entry["qpf_allday"]["mm"] > 0 else 0
measures.append(measure)
elif request == "forecast_snow":
for entry in parsed_json['forecast']['simpleforecast'][
'forecastday'][:forecast_max_entries]:
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:avg"
measure["timestamp"] = utils.day_start(
utils.timezone(int(entry["date"]["epoch"])))
measure["value"] = entry["snow_allday"]["cm"] * 10 if entry[
"snow_allday"]["cm"] > 0 else 0
measures.append(measure)
elif request == "forecast_temperature":
for entry in parsed_json['forecast']['simpleforecast'][
'forecastday'][:forecast_max_entries]:
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:min"
measure["value"] = int(entry["low"]["celsius"])
measure["timestamp"] = utils.day_start(
utils.timezone(int(entry["date"]["epoch"])))
measures.append(measure)
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:max"
measure["value"] = int(entry["high"]["celsius"])
measure["timestamp"] = utils.day_start(
utils.timezone(int(entry["date"]["epoch"])))
measures.append(measure)
elif request == "record_temperature":
measure["key"] = sensor["sensor_id"] + ":day:min"
measure["value"] = int(
parsed_json['almanac']['temp_low']['record']['C'])
measure["timestamp"] = utils.day_start(utils.now())
measures.append(measure)
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:max"
measure["value"] = int(
parsed_json['almanac']['temp_high']['record']['C'])
measure["timestamp"] = utils.day_start(utils.now())
measures.append(measure)
elif request == "record_temperature_year":
measure["key"] = sensor["sensor_id"] + ":day:min"
measure["value"] = int(
parsed_json['almanac']['temp_low']['recordyear'])
measure["timestamp"] = utils.day_start(utils.now())
measures.append(measure)
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:max"
measure["value"] = int(
parsed_json['almanac']['temp_high']['recordyear'])
measure["timestamp"] = utils.day_start(utils.now())
measures.append(measure)
elif request == "normal_temperature":
measure["key"] = sensor["sensor_id"] + ":day:min"
measure["value"] = int(
parsed_json['almanac']['temp_low']['normal']['C'])
measure["timestamp"] = utils.day_start(utils.now())
measures.append(measure)
measure = {}
measure["key"] = sensor["sensor_id"] + ":day:max"
measure["value"] = int(
parsed_json['almanac']['temp_high']['normal']['C'])
measure["timestamp"] = utils.day_start(utils.now())
measures.append(measure)
elif request == "rain":
measure["key"] = sensor["sensor_id"] + ":day:avg"
date_dict = parsed_json['history']['dailysummary'][0]['date']
date = datetime.datetime.strptime(
date_dict["mday"] + "-" + date_dict["mon"] + "-" +
date_dict["year"], "%d-%m-%Y")
measure["timestamp"] = utils.timezone(
int(time.mktime(date.timetuple())))
measure["value"] = float(
parsed_json['history']['dailysummary'][0]['precipm'])
measures.append(measure)
elif request == "snow":
measure["key"] = sensor["sensor_id"] + ":day:avg"
date_dict = parsed_json['history']['dailysummary'][0]['date']
date = datetime.datetime.strptime(
date_dict["mday"] + "-" + date_dict["mon"] + "-" +
date_dict["year"], "%d-%m-%Y")
measure["timestamp"] = utils.timezone(
int(time.mktime(date.timetuple())))
measure["value"] = float(
parsed_json['history']['dailysummary'][0]
['precipm']) if utils.is_number(
parsed_json['history']['dailysummary'][0]['precipm']) else 0
measures.append(measure)
else:
raise Exception("invalid request " + str(request))
# append the measure and return it
return measures
def run():
if not plugin_conf["enabled"]: return
log.debug("[" + __name__ + "] listening for UDP datagrams on port " +
str(plugin_conf['port_listen']))
# bind to the network
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.bind(("", plugin_conf['port_listen']))
while True:
try:
# new data arrives
data, addr = sock.recvfrom(1024)
log.debug("[" + __name__ + "] received " + data)
data = json.loads(data)
if data["type"] != "WirelessMessage": continue
# check if it belongs to a registered sensor
if data["id"] not in nodes: continue
node = nodes[data["id"]]
# for each measure
for message in data["data"]:
if message == "STARTED":
if default_measure not in node: continue
sensor = node[default_measure]
log.info("[" + sensor["module_id"] + "][" +
sensor["sensor_id"] + "] has just started")
# ACK a started message
tx(sensor, "ACK", True)
# initialize
init(sensor)
if message == "AWAKE":
if default_measure not in node: continue
sensor = node[default_measure]
# send a message if there is something in the queue
if data["id"] in queue and len(queue[data["id"]]) > 0:
tx(sensor, queue[data["id"]])
queue[data["id"]] = []
# put it to sleep again
sleep(sensor)
# other messages can be a measure from the sensor
measures = []
# for each registered measure for this node_id
for measure, sensor in node.iteritems():
# skip if not a registered measure
if not message.startswith(measure): continue
measure_data = {}
# generate the timestamp
date = datetime.datetime.strptime(
data["timestamp"], "%d %b %Y %H:%M:%S +0000")
measure_data["timestamp"] = utils.timezone(
utils.timezone(int(time.mktime(date.timetuple()))))
measure_data["key"] = sensor["sensor_id"]
# strip out the measure from the value
measure_data["value"] = utils.normalize(
message.replace(measure, ""), conf["constants"]
["formats"][sensor["format"]]["formatter"])
measures.append(measure_data)
sensors.store(sensor, measures)
except Exception, e:
log.warning("unable to parse " + str(data) + ": " +
utils.get_exception(e))