def upgrade():
db = DbHelper()
with db.session_scope():
db.add_user_account_with_person('Anonymous', 'Anonymous', 'Rest', 'Anonymous')
del db
pass
def __init__(self, json_path):
""" Init tool
@param plugin_name : plugin name
"""
self._db = DbHelper()
try:
self.pkg = PackageJson(path = json_path).json
except PackageException as exp:
print("Error in json file:")
print( exp.value )
exit()
except:
print(str(traceback.format_exc()))
return
print("Json file OK")
# check type == plugin
if self.pkg["identity"]["type"] not in ["plugin", "external"]:
print("Error : this package type is not recognized")
exit()
# check if json version is at least 2
if self.pkg['json_version'] < 2:
print("Error : this package is to old for this version of domogik")
exit()
def __init__(self, log, queue, conv, pub):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._queue = queue
self._conv = conv
self._pub = pub
def test_user_config_file(user_home, user_entry):
info("Check user config file contents")
import ConfigParser
config = ConfigParser.ConfigParser()
config.read("/etc/domogik/domogik.cfg")
#check [domogik] section
dmg = dict(config.items('domogik'))
database = dict(config.items('database'))
admin = dict(config.items('admin'))
butler = dict(config.items('butler'))
backup = dict(config.items('backup'))
ok("Config file correctly loaded")
info("Parse [domogik] section")
import domogik
#Check ix xpl port is not used
_check_port_availability("0.0.0.0", 3865, udp = True)
ok("xPL hub IP/port is not bound by anything else")
parent_conn, child_conn = Pipe()
p = Process(target=_test_user_can_write, args=(child_conn, dmg['log_dir_path'],user_entry,))
p.start()
p.join()
assert parent_conn.recv(), "The directory %s for log does not exist or does not have right permissions" % dmg['log_dir_path']
assert dmg['log_level'] in ['debug','info','warning','error','critical'], "The log_level parameter does not have a good value. Must \
be one of debug,info,warning,error,critical"
### obsolete
#if not os.path.isdir(dmg['src_prefix'] + '/share/domogik'):
# try:
# f = os.listdir("%s/share/domogik" % dmg['src_prefix'])
# f.close()
# except OSError:
# fail("Can't access %s/share/domogik. Check %s is available for domogik user (if you are in development mode, be sure the directory which contains the sources is available for domogik user)." % (dmg['src_prefix'],dmg['src_prefix']))
# exit()
ok("[domogik] section seems good")
# check [database] section
info("Parse [database] section")
assert database['type'] == 'mysql', "Only mysql database type is supported at the moment"
uid = user_entry.pw_uid
os.setreuid(0,uid)
old_home = os.environ['HOME']
os.environ['HOME'] = user_home
from domogik.common.database import DbHelper
d = DbHelper()
os.setreuid(0,0)
os.environ['HOME'] = old_home
assert d.get_engine() != None, "Engine is not set, it seems something went wrong during connection to the database"
ok("[database] section seems good")
# Check [admin] section
info("Parse [admin] section")
for ipadd in get_ip_for_interfaces(admin['interfaces'].split(",")):
_check_port_availability(ipadd, admin['port'])
ok("Admin server IP/port is not bound by anything else")
def __init__(self, log=None, trigger=None, cond=None, params=None):
AbstractTest.__init__(self, log, trigger, cond, params)
self._sensorId = params
self.set_description("Check The value for a sensor with id {0}".format(
self._sensorId))
self.log = log
self._db = DbHelper()
self._res = None
self._dataType = None
self._dt_parent = None
# get info from db
with self._db.session_scope():
sensor = self._db.get_sensor(self._sensorId)
if sensor is not None:
self._res = sensor.last_value
self._dataType = sensor.data_type
# find the parent dt type
if cond:
dt_parent = self._dataType
while 'parent' in cond.datatypes[
dt_parent] and cond.datatypes[dt_parent]['parent'] != None:
dt_parent = cond.datatypes[dt_parent]['parent']
self._dt_parent = dt_parent
# set new val
self._res = self._convert(self._res)
# start the thread
self._event = Event()
self._fetch_thread = Thread(target=self._fetch, name="pollthread")
self._fetch_thread.start()
def _callback(self, message):
""" Callback for the xpl message
@param message : the Xpl message received
"""
self._log_stats.debug("Stat received for device {0}." \
.format(self._dev['name']))
current_date = calendar.timegm(time.gmtime())
stored_value = None
try:
# find what parameter to store
for param in self._stat.params:
# self._log_stats.debug("Checking param {0}".format(param))
if param.sensor_id is not None and param.static is False:
if param.key in message.data:
value = message.data[param.key]
# self._log_stats.debug( \
# "Key found {0} with value {1}." \
# .format(param.key, value))
store = True
if param.ignore_values:
if value in eval(param.ignore_values):
self._log_stats.debug( \
"Value {0} is in the ignore list {0}, so not storing." \
.format(value, param.ignore_values))
store = False
if store:
# check if we need a conversion
if self._sen.conversion is not None and self._sen.conversion != '':
value = call_package_conversion(\
self._log_stats, \
self._dev['client_id'], \
self._sen.conversion, \
value)
self._log_stats.info( \
"Storing stat for device '{0}' ({1}) and sensor'{2}' ({3}): key '{4}' with value '{5}' after conversion." \
.format(self._dev['name'], self._dev['id'], self._sen.name, self._sen.id, param.key, value))
# do the store
stored_value = value
my_db = DbHelper()
with my_db.session_scope():
my_db.add_sensor_history(\
param.sensor_id, \
value, \
current_date)
del(my_db)
else:
self._log_stats.debug("Don't need to store this value")
#else:
# self._log_stats.debug("Key not found in message data")
#else:
# self._log_stats.debug("No sensor attached")
except:
self._log_stats.error(traceback.format_exc())
# publish the result
self._pub.send_event('device-stats', \
{"timestamp" : current_date, \
"device_id" : self._dev['id'], \
"sensor_id" : self._sen.id, \
"stored_value" : stored_value})
def __init__(self, log, queue, storeQueue):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._queue = queue
self._queue_store = storeQueue
# on startup, load the device parameters
self.on_device_changed()
def __init__(self):
""" Initiate DbHelper, Logs and config
"""
XplPlugin.__init__(self, 'xplgw', log_prefix="core_")
self.add_mq_sub('client.conversion')
self.add_mq_sub('client.list')
self.add_mq_sub('client.sensor')
self.add_mq_sub('device.update')
self.log.info(u"XPL manager initialisation...")
self._db = DbHelper()
self.pub = MQPub(zmq.Context(), 'xplgw')
# some initial data sets
self.client_xpl_map = {}
self.client_conversion_map = {}
self._db_sensors = {}
self._db_xplstats = {}
# load devices informations
self._reload_devices()
self._reload_commands()
self._reload_xpl_stats()
# queue to store the message that needs to be ahndled for sensor checking
self._sensor_queue = queue.Queue()
# queue to handle the sensor storage
self._sensor_store_queue = queue.Queue()
# all command handling params
# _lock => to be sure to be thread safe
# _dict => uuid to xplstat translationg
# _pkt => received messages to check
self._cmd_lock_d = threading.Lock()
self._cmd_dict = {}
self._cmd_lock_p = threading.Lock()
self._cmd_pkt = {}
# load some initial data from manager and db
self._load_client_to_xpl_target()
self._load_conversions()
# create a general listener
self._create_xpl_trigger()
# start handling the xplmessages
self._x_thread = self._XplSensorThread(\
self.log, self._sensor_queue, \
self._sensor_store_queue)
self._x_thread.start()
# start handling the command reponses in a thread
self._c_thread = self._XplCommandThread(\
self.log, self._db, self._cmd_lock_d, \
self._cmd_lock_p, self._cmd_dict, self._cmd_pkt, self.pub)
self._c_thread.start()
# start the sensor storage thread
self._s_thread = self._SensorStoreThread(\
self._sensor_store_queue, self.log, \
self._get_conversion_map, self.pub)
self._s_thread.start()
# start the sensorthread
self.ready()
def __init__(self, queue, log, get_conversion_map, pub):
threading.Thread.__init__(self)
self._log = log
self._db = DbHelper()
self._conv = get_conversion_map
self._queue = queue
self._pub = pub
# on startup, load the device parameters
self.on_device_changed()
def __init__(self, log):
""" Create ScenarioManager instance
@param log : Logger instance
"""
# Keep list of conditions as name : instance
self._instances = {}
# an instance of the logger
self.log = log
# load all scenarios from the DB
self._db = DbHelper()
self.load_scenarios()
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug("Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (
nb_test, DATABASE_CONNECTION_NUM_TRY)
print(msg)
self.log.error(msg)
msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT
print(msg)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
print(msg)
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
print(msg)
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error("Error while starting database engine : %s" %
traceback.format_exc())
self.force_leave()
return
Listener(self._request_config_cb, self.myxpl, {
'schema': 'domogik.config',
'xpltype': 'xpl-cmnd'
})
self.enable_hbeat()
IOLoop.instance().start()
def start_http(self):
""" Start HTTP Server
"""
self.log.info(u"Start HTTP Server on %s:%s..." % (self.interfaces, self.port))
# logger
for log in self.log.handlers:
admin_app.logger.addHandler(log)
admin_app.zmq_context = self.zmq
admin_app.db = DbHelper()
admin_app.datatypes = self.datatypes
tapp = Application([
(r"/ws", AdminWebSocket),
(r".*", FallbackHandler, dict(fallback=WSGIContainer(admin_app)))
])
# create the server
# for ssl, extra parameter to HTTPServier init
if self.use_ssl is True:
ssl_options = {
"certfile": self.cert_file,
"keyfile": self.key_file,
}
self.http_server = HTTPServer(tapp, ssl_options=ssl_options)
else:
self.http_server = HTTPServer(tapp)
# listen on the interfaces
if self.interfaces != "":
intf = self.interfaces.split(',')
for ip in get_ip_for_interfaces(intf):
self.http_server.listen(int(self.port), address=ip)
else:
self.http_server.bind(int(self.port))
self.http_server.start(1)
return
def start_http(self):
""" Start HTTP Server
"""
self.log.info(u"Start HTTP Server on {0}:{1}...".format(
self.interfaces, self.port))
# logger
for log in self.log.handlers:
admin_app.logger.addHandler(log)
admin_app.zmq_context = self.zmq
admin_app.db = DbHelper()
admin_app.datatypes = self.datatypes
admin_app.clean_json = self.clean_json
admin_app.rest_auth = self.rest_auth
admin_app.apiversion = REST_API_VERSION
admin_app.use_ssl = self.use_ssl
admin_app.interfaces = self.interfaces
admin_app.port = self.port
admin_app.hostname = self.get_sanitized_hostname()
admin_app.resources_directory = self.get_resources_directory()
admin_app.packages_directory = self.get_packages_directory()
admin_app.publish_directory = self.get_publish_directory(
) # publish directory for all packages
publisher = Publisher()
tapp = Application([(r"/ws", Subscription, dict(publisher=publisher)),
(r".*", FallbackHandler,
dict(fallback=WSGIContainer(admin_app)))])
# create the server
# for ssl, extra parameter to HTTPServier init
if self.use_ssl is True:
ssl_options = {
"certfile": self.cert_file,
"keyfile": self.key_file,
}
print(ssl_options)
self.http_server = HTTPServer(tapp, ssl_options=ssl_options)
else:
self.http_server = HTTPServer(tapp)
# listen on the interfaces
if self.interfaces != "":
# value can be : lo, eth0, ...
# or also : '*' to catch all interfaces, whatever they are
intf = self.interfaces.split(',')
self.log.info(
"The admin will be available on the below addresses : ")
num_int = 0
for ip in get_ip_for_interfaces(intf, log=self.log):
self.log.info(" - {0}:{1} [BIND]".format(ip, self.port))
self.http_server.listen(int(self.port), address=ip)
num_int += 1
if num_int == 0:
self.log.error(
"The admin is not configured to use any working network interface! Please check configuration!!!!!!"
)
else:
self.http_server.bind(int(self.port))
self.http_server.start(0)
yield [publisher.publishToMQ(), publisher.publishToWS()]
def __init__(self):
""" Initiate DbHelper, Logs and config
"""
XplPlugin.__init__(self, 'xplgw', log_prefix = "")
MQAsyncSub.__init__(self, self.zmq, 'xplgw', ['client.conversion', 'client.list'])
self.log.info(u"XPL manager initialisation...")
self._db = DbHelper()
self.pub = MQPub(zmq.Context(), 'xplgw')
self.stats = None
self.client_xpl_map = {}
self.client_conversion_map = {}
self._load_client_to_xpl_target()
self._load_conversions()
self.load()
self.ready()
def __init__(self, handler_params, xpl):
"""
@param handler_params : The server params
@param xpl : A xPL Manager instance
"""
try:
self.myxpl = xpl
# logging initialization
log = logger.Logger('rest-stat')
self._log_stats = log.get_logger('rest-stat')
self._log_stats.info("Rest Stat Manager initialisation...")
# create the dbHelper
self._db = DbHelper()
### Rest data
self.handler_params = handler_params
self.handler_params.append(self._log_stats)
self.handler_params.append(self._db)
self._event_requests = self.handler_params[0]._event_requests
self.get_exception = self.handler_params[0].get_exception
self.stats = None
except :
self._log_stats.error("%s" % traceback.format_exc())
def __init__(self):
'''
Initialize database and xPL connection
'''
Plugin.__init__(self, 'dbmgr', log_prefix='core_')
# Already done in Plugin
#MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug(u"Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
with self._db.session_scope():
# TODO : move in a function and use it (also used in dbmgr)
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(
nb_test, DATABASE_CONNECTION_NUM_TRY,
traceback.format_exc())
self.log.error(msg)
msg = "Waiting for {0} seconds".format(
DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error(
u"Error while starting database engine : {0}".format(
traceback.format_exc()))
self.force_leave()
return
self.ready()
def __init__(self, log, queue, storeQueue):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._queue = queue
self._queue_store = storeQueue
# on startup, load the device parameters
self.on_device_changed()
def _request_config_cb(self, message):
'''
Callback to receive a request for some config stuff
@param message : the xPL message
'''
#try:
self._db = DbHelper(engine=self._engine)
techno = message.data['plugin']
hostname = message.data['hostname']
key = message.data['key']
msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if "value" in message.data:
new_value = message.data['value']
else:
new_value = None
if "element" in message.data:
element = message.data['element']
else:
element = None
msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
# Set configuration
if new_value:
msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, new_value)
print msg
self.log.debug(msg)
self._set_config(techno, hostname, key, new_value)
# Send configuration
else:
msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if element:
msg = "Request h=%s, t=%s, k=%s (send if element)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(techno, hostname, key, self._fetch_elmt_config(techno, element, key), element)
else:
msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if not key:
msg = "Request h=%s, t=%s, k=%s (send if not key)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
keys = self._fetch_techno_config(techno, hostname, key).keys()
values = self._fetch_techno_config(techno, hostname, key).values()
self._send_config(techno, hostname, keys, values)
else:
msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(techno, hostname, key, self._fetch_techno_config(techno, hostname, key))
def __init__(self):
""" Initiate DbHelper, Logs and config
"""
XplPlugin.__init__(self, 'xplgw', log_prefix="core_")
self.add_mq_sub('client.conversion')
self.add_mq_sub('client.list')
self.add_mq_sub('client.sensor')
self.add_mq_sub('device.update')
self.log.info(u"XPL manager initialisation...")
self._db = DbHelper()
self.pub = MQPub(zmq.Context(), 'xplgw')
# some initial data sets
self.client_xpl_map = {}
self.client_conversion_map = {}
self._db_sensors = {}
self._db_xplstats = {}
# load devices informations
self._reload_devices()
self._reload_commands()
self._reload_xpl_stats()
# queue to store the message that needs to be ahndled for sensor checking
self._sensor_queue = queue.Queue()
# queue to handle the sensor storage
self._sensor_store_queue = queue.Queue()
# all command handling params
# _lock => to be sure to be thread safe
# _dict => uuid to xplstat translationg
# _pkt => received messages to check
self._cmd_lock_d = threading.Lock()
self._cmd_dict = {}
self._cmd_lock_p = threading.Lock()
self._cmd_pkt = {}
# load some initial data from manager and db
self._load_client_to_xpl_target()
self._load_conversions()
# create a general listener
self._create_xpl_trigger()
# start handling the xplmessages
self._x_thread = self._XplSensorThread(\
self.log, self._sensor_queue, \
self._sensor_store_queue)
self._x_thread.start()
# start handling the command reponses in a thread
self._c_thread = self._XplCommandThread(\
self.log, self._db, self._cmd_lock_d, \
self._cmd_lock_p, self._cmd_dict, self._cmd_pkt, self.pub)
self._c_thread.start()
# start the sensor storage thread
self._s_thread = self._SensorStoreThread(\
self._sensor_store_queue, self.log, \
self._get_conversion_map, self.pub)
self._s_thread.start()
# start the sensorthread
self.ready()
def __init__(self, log, db, lock_d, lock_p, dic, pkt, pub):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._lock_d = lock_d
self._lock_p = lock_p
self._dict = dic
self._pkt = pkt
self._pub = pub
def __init__(self, queue, log, get_conversion_map, pub):
threading.Thread.__init__(self)
self._log = log
self._db = DbHelper()
self._conv = get_conversion_map
self._queue = queue
self._pub = pub
# on startup, load the device parameters
self.on_device_changed()
def _callback(self, message):
""" Callback for the xpl message
@param message : the Xpl message received
"""
my_db = DbHelper()
self._log_stats.debug("Stat received for device %s." \
% (self._dev.name))
current_date = calendar.timegm(time.gmtime())
device_data = []
try:
# find what parameter to store
for p in self._stat.params:
if p.sensor_id is not None:
if p.key in message.data:
value = message.data[p.key]
self._log_stats.debug("Key found %s with value %s." \
% (p.key, value))
store = True
if p.ignore_values:
if value in eval(p.ignore_values):
self._log_stats.debug("Value %s is in the ignore list %s, so not storing." \
% (value, p.ignore_values))
store = False
if store:
# check if we need a conversion
if self._sen.conversion is not None and self._sen.conversion != '':
value = call_package_conversion(\
self._log_stats, self._dev.device_type.plugin_id, \
self._sen.conversion, value)
self._log_stats.debug("Key found %s with value %s after conversion." \
% (p.key, value))
# do the store
device_data.append({"value" : value, "sensor": p.sensor_id})
my_db.add_sensor_history(p.sensor_id, value, current_date)
except:
error = "Error when processing stat : %s" % traceback.format_exc()
print("==== Error in Stats ====")
print(error)
print("========================")
self._log_stats.error(error)
# put data in the event queue
self._event_requests.add_in_queues(self._dev.id,
{"timestamp" : current_date, "device_id" : self._dev.id, "data" : device_data})
del(my_db)
def __init__(self, log):
""" Create ScenarioManager instance
@param log : Logger instance
"""
# Keep list of conditions as name : instance
self._instances = {}
# an instance of the logger
self.log = log
# load all scenarios from the DB
self._db = DbHelper()
self.load_scenarios()
def __init__(self):
""" Initiate DbHelper, Logs and config
"""
XplPlugin.__init__(self, 'xplgw')
self.log.info(u"XPL manager initialisation...")
self._db = DbHelper()
self.pub = MQPub(zmq.Context(), 'xplgw')
self.stats = None
self.client_xpl_map = {}
self._load_client_to_xpl_target()
self.load()
self.ready()
def __init__(self, log = None, trigger = None, cond = None, params = None):
AbstractTest.__init__(self, log, trigger, cond, params)
self._subMessages.append( 'device-stats' )
self._sensorId = params
self.set_description(u"Check if value/date changes for a sensor with id {0}".format(self._sensorId))
self.log = log
self._db = DbHelper()
self._res = None
self._dataType = None
self._dt_parent = None
# get initital info from db
with self._db.session_scope():
sensor = self._db.get_sensor(self._sensorId)
if sensor is not None:
self._res = sensor.last_value
self._dataType = sensor.data_type
# find the parent dt type
if cond:
dt_parent = self._dataType
while 'parent' in cond.datatypes[dt_parent] and cond.datatypes[dt_parent]['parent'] != None:
dt_parent = cond.datatypes[dt_parent]['parent']
self._dt_parent = dt_parent
def start_http(self):
""" Start HTTP Server
"""
self.log.info(u"Start HTTP Server on %s:%s..." %
(self.interfaces, self.port))
# logger
for log in self.log.handlers:
urlHandler.logger.addHandler(log)
# db access
urlHandler.db = DbHelper()
# needed for status
urlHandler.apiversion = self._rest_api_version
urlHandler.use_ssl = self.use_ssl
urlHandler.hostname = self.get_sanitized_hostname()
urlHandler.clean_json = self.clean_json
urlHandler.zmq_context = self.zmq
# handler for getting the paths
urlHandler.resources_directory = self.get_resources_directory()
# create the server
# for ssl, extra parameter to HTTPServier init
if self.use_ssl:
ssl_options = {
"certfile": self.cert_file,
"keyfile": self.key_file,
}
self.http_server = HTTPServer(WSGIContainer(urlHandler),
ssl_options=ssl_options)
else:
self.http_server = HTTPServer(WSGIContainer(urlHandler))
# listen on the interfaces
if self.interfaces != "":
intf = self.interfaces.split(',')
self.log.info(
"REST server will be available on the below addresses : ")
num_int = 0
for ip in get_ip_for_interfaces(intf, log=self.log):
self.log.info(" - {0}:{1} [BIND]".format(ip, self.port))
self.http_server.listen(int(self.port), address=ip)
num_int += 1
if num_int == 0:
self.log.error(
"The rest server is not configured to use any working network interface! Please check configuration!!!!!!"
)
else:
self.http_server.bind(int(self.port))
self.http_server.start(1)
return
class AbstractSensorTest(AbstractTest):
def __init__(self, log = None, trigger = None, cond = None, params = None):
AbstractTest.__init__(self, log, trigger, cond, params)
self._subMessages.append( 'device-stats' )
self._sensorId = params
self.set_description(u"Check if value/date changes for a sensor with id {0}".format(self._sensorId))
self.log = log
self._db = DbHelper()
self._res = None
self._dataType = None
self._dt_parent = None
# get initital info from db
with self._db.session_scope():
sensor = self._db.get_sensor(self._sensorId)
if sensor is not None:
self._res = sensor.last_value
self._dataType = sensor.data_type
# find the parent dt type
if cond:
dt_parent = self._dataType
while 'parent' in cond.datatypes[dt_parent] and cond.datatypes[dt_parent]['parent'] != None:
dt_parent = cond.datatypes[dt_parent]['parent']
self._dt_parent = dt_parent
def destroy(self):
""" Destroy fetch thread
"""
AbstractTest.destroy(self)
def on_message(self, did, msg):
"""Receive message from MQ sub"""
if self._sensorId:
if 'sensor_id' in msg:
if int(msg['sensor_id']) == int(self._sensorId):
self.log.debug(u"{0} : received MQ message : {1}. Please notice that if a scenario which use several of this sensor block is currently being evaluated, only one value is processed (in case the sensor values change too fast!)".format(self.__class__.__name__, msg))
self.handle_message(did, msg)
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug("Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (nb_test, DATABASE_CONNECTION_NUM_TRY)
print(msg)
self.log.error(msg)
msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT
print(msg)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
print(msg)
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
print(msg)
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error("Error while starting database engine : %s" % traceback.format_exc())
self.force_leave()
return
Listener(self._request_config_cb, self.myxpl, {'schema': 'domogik.config', 'xpltype': 'xpl-cmnd'})
self.enable_hbeat()
IOLoop.instance().start()
def __init__(self, json_path):
""" Init tool
@param plugin_name : plugin name
"""
self._db = DbHelper()
try:
self.pkg = PackageJson(path = json_path).json
except:
print(str(traceback.format_exc()))
return
print("Json file OK")
# check type == plugin
if self.pkg["identity"]["type"] not in ["plugin", "external"]:
print("Error : this package type is not recognized")
exit()
def __init__(self):
'''
Initialize database and xPL connection
'''
Plugin.__init__(self, 'dbmgr', log_prefix='core_')
# Already done in Plugin
#MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug(u"Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
with self._db.session_scope():
# TODO : move in a function and use it (also used in dbmgr)
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc())
self.log.error(msg)
msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc()))
self.force_leave()
return
self.ready()
def __init__(self, log):
""" Create ScenarioManager instance
@param log : Logger instance
"""
# Keep uuid <-> instance mapping for tests and actions
self._tests_mapping = {}
self._actions_mapping = {}
# Keep list of conditions as name : instance
self._conditions = {}
# Keep list of actions uuid linked to a condition as name : [uuid1, uuid2, ... ]
self._conditions_actions = {}
# an instance of the logger
self.log = log
# As we lazy-load all the tests/actions in __instanciate
# we need to keep the module in a list so that we don't need to reload them
# every time.
self._test_cache = {}
self._action_cache = {}
# load all scenarios from the DB
self._db = DbHelper()
self.load_scenarios()
def start_http(self):
""" Start HTTP Server
"""
self.log.info(u"Start HTTP Server on %s:%s..." % (self.interfaces, self.port))
# logger
for log in self.log.handlers:
urlHandler.logger.addHandler(log)
# db access
urlHandler.db = DbHelper()
# needed for status
urlHandler.apiversion = self._rest_api_version
urlHandler.use_ssl = self.use_ssl
urlHandler.hostname = self.get_sanitized_hostname()
urlHandler.clean_json = self.clean_json
# reload statsmanager helper
urlHandler.reload_stats = self.reload_stats
urlHandler.zmq_context = self.zmq
# handler for getting the paths
urlHandler.resources_directory = self.get_resources_directory()
# create the server
# for ssl, extra parameter to HTTPServier init
if self.use_ssl:
ssl_options = {
"certfile": self.cert_file,
"keyfile": self.key_file,
}
self.http_server = HTTPServer(WSGIContainer(urlHandler), ssl_options=ssl_options)
else:
self.http_server = HTTPServer(WSGIContainer(urlHandler))
# listen on the interfaces
if self.interfaces != "":
intf = self.interfaces.split(',')
for ip in get_ip_for_interfaces(intf):
self.http_server.listen(int(self.port), address=ip)
else:
self.http_server.bind(int(self.port))
self.http_server.start(1)
return
class _SensorThread(threading.Thread):
""" SensorThread class
Class that will handle the sensor storage in a seperated thread
This will get messages from the SensorQueue
"""
def __init__(self, log, queue, conv, pub):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._queue = queue
self._conv = conv
self._pub = pub
def _find_storeparam(self, item):
#print("ITEM = {0}".format(item['msg']))
found = False
tostore = []
for xplstat in self._db.get_all_xpl_stat():
sensors = 0
matching = 0
statics = 0
if xplstat.schema == item["msg"].schema:
#print(" XPLSTAT = {0}".format(xplstat))
# we found a possible xplstat
# try to match all params and try to find a sensorid and a vlaue to store
for param in xplstat.params:
#print(" PARAM = {0}".format(param))
### Caution !
# in case you, who are reading this, have to debug something like that :
# 2015-08-16 22:04:26,190 domogik-xplgw INFO Storing stat for device 'Garage' (6) and sensor 'Humidity' (69): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,306 domogik-xplgw INFO Storing stat for device 'Salon' (10) and sensor 'Humidity' (76): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,420 domogik-xplgw INFO Storing stat for device 'Chambre d'Ewan' (11) and sensor 'Humidity' (80): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,533 domogik-xplgw INFO Storing stat for device 'Chambre des parents' (12) and sensor 'Humidity' (84): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,651 domogik-xplgw INFO Storing stat for device 'Chambre de Laly' (13) and sensor 'Humidity' (88): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,770 domogik-xplgw INFO Storing stat for device 'Entrée' (17) and sensor 'Humidity' (133): key 'current' with value '53' after conversion.
#
# which means that for a single xPL message, the value is stored in several sensors (WTF!!! ?)
# It can be related to the fact that the device address key is no more corresponding between the plugin (info.json and xpl sent by python) and the way the device was create in the databse
# this should not happen, but in case... well, we may try to find a fix...
if param.key in item["msg"].data and param.static:
statics = statics + 1
if param.multiple is not None and len(param.multiple) == 1 and item["msg"].data[param.key] in param.value.split(param.multiple):
matching = matching + 1
elif item["msg"].data[param.key] == param.value:
matching = matching + 1
# now we have a matching xplstat, go and find all sensors
if matching == statics:
#print(" MATHING !!!!!")
for param in xplstat.params:
if param.key in item["msg"].data and not param.static:
#print(" ===> TOSTORE !!!!!!!!! : {0}".format({'param': param, 'value': item["msg"].data[param.key]}))
tostore.append( {'param': param, 'value': item["msg"].data[param.key]} )
if len(tostore) > 0:
found = True
if found:
return (found, tostore)
else:
return False
def run(self):
while True:
try:
item = self._queue.get()
self._log.debug(u"Getting item from the sensorQueue, current length = {0}".format(self._queue.qsize()))
# if clientid is none, we don't know this sender so ignore
# TODO check temp disabled until external members are working
#if item["clientId"] is not None:
if True:
with self._db.session_scope():
fdata = self._find_storeparam(item)
if fdata:
#// ICI !!!
self._log.debug(u"Found a matching sensor, so starting the storage procedure. Sensor : {0}".format(fdata))
for data in fdata[1]:
value = data['value']
storeparam = data['param']
current_date = calendar.timegm(time.gmtime())
store = True
if storeparam.ignore_values:
if value in eval(storeparam.ignore_values):
self._log.debug(u"Value {0} is in the ignore list {0}, so not storing.".format(value, storeparam.ignore_values))
store = False
if store:
# get the sensor and dev
sen = self._db.get_sensor(storeparam.sensor_id)
dev = self._db.get_device(sen.device_id)
# check if we need a conversion
if sen.conversion is not None and sen.conversion != '':
if dev['client_id'] in self._conv and sen.conversion in self._conv[dev['client_id']]:
self._log.debug( \
u"Calling conversion {0}".format(sen.conversion))
exec(self._conv[dev['client_id']][sen.conversion])
value = locals()[sen.conversion](value)
self._log.info( \
u"Storing stat for device '{0}' ({1}) and sensor '{2}' ({3}): key '{4}' with value '{5}' after conversion." \
.format(dev['name'], dev['id'], sen.name, sen.id, storeparam.key, value))
# do the store
try:
self._db.add_sensor_history(\
storeparam.sensor_id, \
value, \
current_date)
except Exception as exp:
self._log.error(u"Error when adding sensor history : {0}".format(traceback.format_exc()))
else:
self._log.debug(u"Don't need to store this value")
# publish the result
self._pub.send_event('device-stats', \
{"timestamp" : current_date, \
"device_id" : dev['id'], \
"sensor_id" : sen.id, \
"stored_value" : value})
except Queue.Empty:
# nothing in the queue, sleep for 1 second
time.sleep(1)
except Exception as exp:
self._log.error(traceback.format_exc())
def _request_config_cb(self, message):
'''
Callback to receive a request for some config stuff
@param message : the xPL message
'''
#try:
self._db = DbHelper(engine=self._engine)
techno = message.data['plugin']
hostname = message.data['hostname']
key = message.data['key']
msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if "value" in message.data:
new_value = message.data['value']
else:
new_value = None
if "element" in message.data:
element = message.data['element']
else:
element = None
msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
# Set configuration
if new_value:
msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key,
new_value)
print msg
self.log.debug(msg)
self._set_config(techno, hostname, key, new_value)
# Send configuration
else:
msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if element:
msg = "Request h=%s, t=%s, k=%s (send if element)" % (
hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(
techno, hostname, key,
self._fetch_elmt_config(techno, element, key), element)
else:
msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname,
techno, key)
print(msg)
self.log.debug(msg)
if not key:
msg = "Request h=%s, t=%s, k=%s (send if not key)" % (
hostname, techno, key)
print(msg)
self.log.debug(msg)
keys = self._fetch_techno_config(techno, hostname,
key).keys()
values = self._fetch_techno_config(techno, hostname,
key).values()
self._send_config(techno, hostname, keys, values)
else:
msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % (
hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(
techno, hostname, key,
self._fetch_techno_config(techno, hostname, key))
class DBConnector(XplPlugin, MQRep):
'''
Manage the connection between database and the xPL stuff
Should be the *only* object along with the StatsManager to access to the database on the core side
'''
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug("Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (
nb_test, DATABASE_CONNECTION_NUM_TRY)
print(msg)
self.log.error(msg)
msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT
print(msg)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
print(msg)
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
print(msg)
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error("Error while starting database engine : %s" %
traceback.format_exc())
self.force_leave()
return
Listener(self._request_config_cb, self.myxpl, {
'schema': 'domogik.config',
'xpltype': 'xpl-cmnd'
})
self.enable_hbeat()
IOLoop.instance().start()
def on_mdp_request(self, msg):
if msg.get_action() == "config.get":
plugin = msg._data['plugin']
hostname = msg._data['hostname']
key = msg._data['key']
if 'element' in msg._data.keys():
element = msg._data['element']
else:
element = None
if element:
self._mdp_reply(plugin, hostname, key,
self._fetch_elmt_config(plugin, element, key),
element)
elif not key:
print 'TODO'
else:
self._mdp_reply(
plugin, hostname, key,
self._fetch_techno_config(plugin, hostname, key))
elif msg._action == "config.set":
print 'TODO'
def _mdp_reply(self, plugin, hostname, key, value, element=None):
msg = MQMessage()
msg.setaction('config.result')
msg.add_data('plugin', plugin)
msg.add_data('hostname', hostname)
msg.add_data('key', key)
msg.add_data('value', value)
if element:
msg.add_data('element', element)
print msg.get()
self.reply(msg.get())
def _request_config_cb(self, message):
'''
Callback to receive a request for some config stuff
@param message : the xPL message
'''
#try:
self._db = DbHelper(engine=self._engine)
techno = message.data['plugin']
hostname = message.data['hostname']
key = message.data['key']
msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if "value" in message.data:
new_value = message.data['value']
else:
new_value = None
if "element" in message.data:
element = message.data['element']
else:
element = None
msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
# Set configuration
if new_value:
msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key,
new_value)
print msg
self.log.debug(msg)
self._set_config(techno, hostname, key, new_value)
# Send configuration
else:
msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if element:
msg = "Request h=%s, t=%s, k=%s (send if element)" % (
hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(
techno, hostname, key,
self._fetch_elmt_config(techno, element, key), element)
else:
msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname,
techno, key)
print(msg)
self.log.debug(msg)
if not key:
msg = "Request h=%s, t=%s, k=%s (send if not key)" % (
hostname, techno, key)
print(msg)
self.log.debug(msg)
keys = self._fetch_techno_config(techno, hostname,
key).keys()
values = self._fetch_techno_config(techno, hostname,
key).values()
self._send_config(techno, hostname, keys, values)
else:
msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % (
hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(
techno, hostname, key,
self._fetch_techno_config(techno, hostname, key))
def _send_config(self, plugin, hostname, key, value, element=None):
'''
Send a config value message for an element's config item
@param plugin : the plugin of the element
@param hostname : hostname
@param element : the name of the element
@param key : the key or list of keys of the config tuple(s) to fetch
@param value : the value or list of values corresponding to the key(s)
'''
msg = "Response h=%s, t=%s, k=%s, v=%s" % (hostname, plugin, key,
value)
print(msg)
self.log.debug(msg)
mess = XplMessage()
mess.set_type('xpl-stat')
mess.set_schema('domogik.config')
mess.add_data({'plugin': plugin})
mess.add_data({'hostname': hostname})
if element:
mess.add_data({'element': element})
# If key/value are lists, then we add a key=value for each item
if isinstance(key, list):
for (_key, _val) in zip(key, value):
mess.add_data({_key: _val})
else:
mess.add_data({key: value})
# mess.set_conf_key('target', plugin)
self.myxpl.send(mess)
def _fetch_elmt_config(self, techno, element, key):
'''
Fetch an element's config value in the database
@param techno : the plugin of the element
@param element : the name of the element
@param key : the key of the config tuple to fetch
'''
#TODO : use the database
vals = {
'x10': {
'a3': {},
'a2': {},
}
}
return vals[techno][element][key]
def _fetch_techno_config(self, techno, hostname, key):
'''
Fetch a plugin global config value in the database
@param techno : the plugin of the element
@param hostname : hostname
@param key : the key of the config tuple to fetch
'''
# This array is here for information only but is not used anymore
# Values are now on the database
#vals = {'x10': {'heyu-cfg-path':'/etc/heyu/x10.conf',
# 'heyu-file-0': 'TTY /dev/ttyUSB0',
# 'heyu-file-1': 'TTY_AUX /dev/ttyUSB0 RFXCOM',
# 'heyu-file-2': 'ALIAS back_door D5 DS10A 0x677'},
# 'global': {'pid-dir-path': '/var/run/'},
# 'onewire': {'temperature_refresh_delay' : '10'},
# 'cidmodem': {'device' : '/dev/ttyUSB1',
# 'nbmaxtry' : '10',
# 'interval' : '15'},
# 'mirror': {'device' : '/dev/hidraw0',
# 'nbmaxtry' : '10',
# 'interval' : '15'},
# 'xbmc_not': {'address' : '192.168.0.20:8080',
# 'delay' : '15',
# 'maxdelay' : '20'},
# 'gagenda': {'email' : "*****@*****.**",
# 'password' : 'XXXXXXXX',
# 'calendarname' : '*****@*****.**',
# 'startup-plugin':'True'},
# 'teleinfo' : {'device' : '/dev/teleinfo',
# 'interval' : '30'},
# 'dawndusk' : {'startup-plugin':'True'},
# 'plcbus' : {'device':'/dev/ttyUSB0'},
# }
self.log.debug("FTC 1")
try:
if key:
self.log.debug("FTC 2")
try:
self.log.debug("Get plg conf for %s / %s / %s" %
(techno, hostname, key))
result = self._db.get_plugin_config(techno, hostname, key)
# tricky loop as workaround for a (sqlalchemy?) bug :
# sometimes the given result is for another plugin/key
# so while we don't get the good data, we loop
# This bug happens rarely
while result.id != techno or \
result.hostname != hostname or \
result.key != key:
self.log.debug("Bad result : %s/%s != %s/%s" %
(result.id, result.key, plugin, key))
result = self._db.get_plugin_config(
techno, hostname, key)
self.log.debug("Get plg conf for %s / %s / %s Result=%s" %
(techno, hostname, key, result))
val = result.value
self.log.debug("Get plg conf for %s / %s / %s = %s" %
(techno, hostname, key, val))
if val == '':
val = "None"
self.log.debug("Get plg conf for %s / %s / %s = %s (2)" %
(techno, hostname, key, val))
return val
except AttributeError:
self.log.debug("Attribute error for %s / %s / %s" %
(techno, hostname, key))
return "None"
else:
self.log.debug("FTC 3")
vals = self._db.list_plugin_config(techno, hostname)
res = {}
for val in vals:
if val == '':
res[val.key] = "None"
else:
res[val.key] = val.value
return res
except:
msg = "No config found h=%s, t=%s, k=%s" % (hostname, techno, key)
print(msg)
self.log.warn(msg)
return "None"
def _set_config(self, plugin, hostname, key, value):
'''
Send a config value message for an element's config item
@param plugin : the plugin of the element
@param hostname : hostname
@param key : the key to set
@param value : the value to set
'''
try:
self._db.set_plugin_config(technology, hostname, key, value)
mess = XplMessage()
mess.set_type('xpl-stat')
mess.set_schema('domogik.config')
mess.add_data({'plugin': plugin})
mess.add_data({'hostname': hostname})
mess.add_data({'key': key})
mess.add_data({'value': value})
self.myxpl.send(mess)
except:
traceback.print_exc()
msg = "Error while setting h=%s, t=%s, k=%s, v=%s" % (
hostname, techno, key, value)
print(msg)
self.log.warn(msg)
return "None"
class XplManager(XplPlugin, MQAsyncSub):
""" Statistics manager
"""
def __init__(self):
""" Initiate DbHelper, Logs and config
"""
XplPlugin.__init__(self, 'xplgw', log_prefix = "")
MQAsyncSub.__init__(self, self.zmq, 'xplgw', ['client.conversion', 'client.list'])
self.log.info(u"XPL manager initialisation...")
self._db = DbHelper()
self.pub = MQPub(zmq.Context(), 'xplgw')
self.stats = None
self.client_xpl_map = {}
self.client_conversion_map = {}
self._load_client_to_xpl_target()
self._load_conversions()
self.load()
self.ready()
def on_mdp_request(self, msg):
# XplPlugin handles MQ Req/rep also
try:
XplPlugin.on_mdp_request(self, msg)
if msg.get_action() == "reload":
self.load()
msg = MQMessage()
msg.set_action( 'reload.result' )
self.reply(msg.get())
elif msg.get_action() == "cmd.send":
self._send_xpl_command(msg)
except:
self.log.error(traceback.format_exc())
def on_message(self, msgid, content):
try:
if msgid == 'client.conversion':
self._parse_conversions(content)
elif msgid == 'client.list':
self._parse_xpl_target(content)
except:
self.log.error(traceback.format_exc())
def _load_client_to_xpl_target(self):
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('client.list.get')
response = cli.request('manager', msg.get(), timeout=10)
if response:
self._parse_xpl_target(response.get_data())
else:
self.log.error(u"Updating client list was not successfull, no response from manager")
def _parse_xpl_target(self, data):
tmp = {}
for cli in data:
tmp[cli] = data[cli]['xpl_source']
self.client_xpl_map = tmp
def _load_conversions(self):
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('client.conversion.get')
response = cli.request('manager', msg.get(), timeout=10)
if response:
self._parse_conversions(response.get_data())
else:
self.log.error(u"Updating client conversion list was not successfull, no response from manager")
def _parse_conversions(self, data):
tmp = {}
for cli in data:
tmp[cli] = data[cli]
self.client_conversion_map = tmp
def _send_xpl_command(self, data):
""" Reply to config.get MQ req
@param data : MQ req message
Needed info in data:
- cmdid => command id to send
- cmdparams => key/value pair of all params needed for this command
"""
with self._db.session_scope():
self.log.info(u"Received new cmd request: {0}".format(data))
failed = False
request = data.get_data()
if 'cmdid' not in request:
failed = "cmdid not in message data"
if 'cmdparams' not in request:
failed = "cmdparams not in message data"
if not failed:
# get the command
cmd = self._db.get_command(request['cmdid'])
if cmd is not None:
if cmd.xpl_command is not None:
xplcmd = cmd.xpl_command
xplstat = self._db.get_xpl_stat(xplcmd.stat_id)
if xplstat is not None:
# get the device from the db
dev = self._db.get_device(int(cmd.device_id))
msg = XplMessage()
if not dev['client_id'] in self.client_xpl_map.keys():
self._load_client_to_xpl_target()
if not dev['client_id'] in self.client_xpl_map.keys():
failed = "Can not fincd xpl source for {0} client_id".format(dev['client_id'])
else:
msg.set_target(self.client_xpl_map[dev['client_id']])
msg.set_source(self.myxpl.get_source())
msg.set_type("xpl-cmnd")
msg.set_schema( xplcmd.schema)
# static params
for p in xplcmd.params:
msg.add_data({p.key : p.value})
# dynamic params
for p in cmd.params:
if p.key in request['cmdparams']:
value = request['cmdparams'][p.key]
# chieck if we need a conversion
if p.conversion is not None and p.conversion != '':
if dev['client_id'] in self.client_conversion_map:
if p.conversion in self.client_conversion_map[dev['client_id']]:
exec(self.client_conversion_map[dev['client_id']][p.conversion])
value = locals()[p.conversion](value)
msg.add_data({p.key : value})
else:
failed = "Parameter ({0}) for device command msg is not provided in the mq message".format(p.key)
if not failed:
# send out the msg
self.log.debug(u"sending xplmessage: {0}".format(msg))
self.myxpl.send(msg)
### Wait for answer
stat_received = 0
if xplstat != None:
# get xpl message from queue
self.log.debug(u"Command : wait for answer...")
sub = MQSyncSub( self.zmq, 'xplgw-command', ['device-stats'] )
stat = sub.wait_for_event()
if stat is not None:
reply = json.loads(stat['content'])
reply_msg = MQMessage()
reply_msg.set_action('cmd.send.result')
reply_msg.add_data('stat', reply)
reply_msg.add_data('status', True)
reply_msg.add_data('reason', None)
self.log.debug(u"mq reply".format(reply_msg.get()))
self.reply(reply_msg.get())
if failed:
self.log.error(failed)
reply_msg = MQMessage()
reply_msg.set_action('cmd.send.result')
reply_msg.add_data('status', False)
reply_msg.add_data('reason', failed)
self.log.debug(u"mq reply".format(reply_msg.get()))
self.reply(reply_msg.get())
def load(self):
""" (re)load all xml files to (re)create _Stats objects
"""
self.log.info(u"Rest Stat Manager loading.... ")
self._db.open_session()
try:
# not the first load : clean
if self.stats != None:
self.log.info(u"reloading")
for stat in self.stats:
self.myxpl.del_listener(stat.get_listener())
### Load stats
# key1, key2 = device_type_id, schema
self.stats = []
created_stats = []
for stat in self._db.get_all_xpl_stat():
# xpl-trig
self.stats.append(self._Stat(self.myxpl, stat, "xpl-trig", \
self.log, self.pub, self.client_conversion_map))
# xpl-stat
self.stats.append(self._Stat(self.myxpl, stat, "xpl-stat", \
self.log, self.pub, self.client_conversion_map))
except:
self.log.error(u"%s" % traceback.format_exc())
self._db.close_session()
self.log.info(u"Loading finished")
class _Stat:
""" This class define a statistic parser and logger instance
Each instance create a Listener and the associated callbacks
"""
def __init__(self, xpl, stat, xpl_type, log, pub, conversions):
""" Initialize a stat instance
@param xpl : A xpl manager instance
@param stat : A XplStat reference
@param xpl-type: what xpl-type to listen for
"""
### Rest data
self._log_stats = log
self._stat = stat
self._pub = pub
self._conv = conversions
### build the filter
params = {'schema': stat.schema, 'xpltype': xpl_type}
for param in stat.params:
if param.static:
params[param.key] = param.value
### start the listener
self._log_stats.info("creating listener for %s" % (params))
self._listener = Listener(self._callback, xpl, params)
def get_listener(self):
""" getter for lsitener object
"""
return self._listener
def _callback(self, message):
""" Callback for the xpl message
@param message : the Xpl message received
"""
self._log_stats.debug( "_callback started for: {0}".format(message) )
db = DbHelper()
current_date = calendar.timegm(time.gmtime())
stored_value = None
try:
# find what parameter to store
for param in self._stat.params:
# self._log_stats.debug("Checking param {0}".format(param))
if param.sensor_id is not None and param.static is False:
if param.key in message.data:
with db.session_scope():
value = message.data[param.key]
# self._log_stats.debug( \
# "Key found {0} with value {1}." \
# .format(param.key, value))
store = True
if param.ignore_values:
if value in eval(param.ignore_values):
self._log_stats.debug( \
"Value {0} is in the ignore list {0}, so not storing." \
.format(value, param.ignore_values))
store = False
if store:
# get the sensor and dev
sen = db.get_sensor(param.sensor_id)
dev = db.get_device(sen.device_id)
# check if we need a conversion
if sen.conversion is not None and sen.conversion != '':
if dev['client_id'] in self._conv:
if sen.conversion in self._conv[dev['client_id']]:
self._log_stats.debug( \
"Calling conversion {0}".format(sen.conversion))
exec(self._conv[dev['client_id']][sen.conversion])
value = locals()[sen.conversion](value)
self._log_stats.info( \
"Storing stat for device '{0}' ({1}) and sensor'{2}' ({3}): key '{4}' with value '{5}' after conversion." \
.format(dev['name'], dev['id'], sen.name, sen.id, param.key, value))
# do the store
stored_value = value
try:
db.add_sensor_history(\
param.sensor_id, \
value, \
current_date)
except:
self._log_stats.error("Error when adding sensor history : {0}".format(traceback.format_exc()))
else:
self._log_stats.debug("Don't need to store this value")
# publish the result
self._pub.send_event('device-stats', \
{"timestamp" : current_date, \
"device_id" : dev['id'], \
"sensor_id" : sen.id, \
"stored_value" : stored_value})
#else:
# self._log_stats.debug("Key not found in message data")
#else:
# self._log_stats.debug("No sensor attached")
except:
self._log_stats.error(traceback.format_exc())
def _callback(self, message):
""" Callback for the xpl message
@param message : the Xpl message received
"""
self._log_stats.debug( "_callback started for: {0}".format(message) )
db = DbHelper()
current_date = calendar.timegm(time.gmtime())
stored_value = None
try:
# find what parameter to store
for param in self._stat.params:
# self._log_stats.debug("Checking param {0}".format(param))
if param.sensor_id is not None and param.static is False:
if param.key in message.data:
with db.session_scope():
value = message.data[param.key]
# self._log_stats.debug( \
# "Key found {0} with value {1}." \
# .format(param.key, value))
store = True
if param.ignore_values:
if value in eval(param.ignore_values):
self._log_stats.debug( \
"Value {0} is in the ignore list {0}, so not storing." \
.format(value, param.ignore_values))
store = False
if store:
# get the sensor and dev
sen = db.get_sensor(param.sensor_id)
dev = db.get_device(sen.device_id)
# check if we need a conversion
if sen.conversion is not None and sen.conversion != '':
if dev['client_id'] in self._conv:
if sen.conversion in self._conv[dev['client_id']]:
self._log_stats.debug( \
"Calling conversion {0}".format(sen.conversion))
exec(self._conv[dev['client_id']][sen.conversion])
value = locals()[sen.conversion](value)
self._log_stats.info( \
"Storing stat for device '{0}' ({1}) and sensor'{2}' ({3}): key '{4}' with value '{5}' after conversion." \
.format(dev['name'], dev['id'], sen.name, sen.id, param.key, value))
# do the store
stored_value = value
try:
db.add_sensor_history(\
param.sensor_id, \
value, \
current_date)
except:
self._log_stats.error("Error when adding sensor history : {0}".format(traceback.format_exc()))
else:
self._log_stats.debug("Don't need to store this value")
# publish the result
self._pub.send_event('device-stats', \
{"timestamp" : current_date, \
"device_id" : dev['id'], \
"sensor_id" : sen.id, \
"stored_value" : stored_value})
#else:
# self._log_stats.debug("Key not found in message data")
#else:
# self._log_stats.debug("No sensor attached")
except:
self._log_stats.error(traceback.format_exc())
class PackageData():
""" Tool to insert necessary data in database
"""
def __init__(self, json_path):
""" Init tool
@param plugin_name : plugin name
"""
self._db = DbHelper()
try:
self.pkg = PackageJson(path = json_path).json
except PackageException as exp:
print("Error in json file:")
print( exp.value )
exit()
except:
print(str(traceback.format_exc()))
return
print("Json file OK")
# check type == plugin
if self.pkg["identity"]["type"] not in ["plugin", "external"]:
print("Error : this package type is not recognized")
exit()
# check if json version is at least 2
if self.pkg['json_version'] < 2:
print("Error : this package is to old for this version of domogik")
exit()
def insert(self):
""" Insert data for plugin
"""
### Plugin
print("plugin %s" % self.pkg["identity"]["id"])
if self._db.get_plugin(self.pkg["identity"]["id"]) == None:
# add if not exists
print("add...")
self._db.add_plugin(self.pkg["identity"]["id"],
self.pkg["identity"]["description"],
self.pkg["identity"]["version"])
else:
# update if exists
print("update...")
self._db.update_plugin(self.pkg["identity"]["id"],
self.pkg["identity"]["description"],
self.pkg["identity"]["version"])
### Device types
for device_type in self.pkg["device_types"].keys():
print("Device type %s" % device_type)
device_type = self.pkg["device_types"][device_type]
if self._db.get_device_type_by_id(device_type["id"]) == None:
# add if not exists
print("add...")
self._db.add_device_type(device_type["id"],
device_type["name"],
self.pkg["identity"]["id"],
device_type["description"])
else:
# update if exists
print("update...")
self._db.update_device_type(device_type["id"],
device_type["name"],
self.pkg["identity"]["id"],
device_type["description"])
class StatsManager:
"""
Listen on the xPL network and keep stats of device and system state
"""
def __init__(self, handler_params, xpl):
"""
@param handler_params : The server params
@param xpl : A xPL Manager instance
"""
try:
self.myxpl = xpl
# logging initialization
log = logger.Logger('rest-stat')
self._log_stats = log.get_logger('rest-stat')
self._log_stats.info("Rest Stat Manager initialisation...")
# create the dbHelper
self._db = DbHelper()
### Rest data
self.handler_params = handler_params
self.handler_params.append(self._log_stats)
self.handler_params.append(self._db)
self._event_requests = self.handler_params[0]._event_requests
self.get_exception = self.handler_params[0].get_exception
self.stats = None
except :
self._log_stats.error("%s" % traceback.format_exc())
def load(self):
""" (re)load all xml files to (re)create _Stats objects
"""
self._log_stats.info("Rest Stat Manager loading.... ")
try:
# not the first load : clean
if self.stats != None:
for x in self.stats:
self.myxpl.del_listener(x.get_listener())
### Load stats
# key1, key2 = device_type_id, schema
self.stats = []
for sen in self._db.get_all_sensor():
self._log_stats.debug(sen)
statparam = self._db.get_xpl_stat_param_by_sensor(sen.id)
if statparam is None:
self._log_stats.error('Corresponding xpl-stat param can not be found for sensor %s' % (sen))
continue
stat = self._db.get_xpl_stat(statparam.xplstat_id)
if stat is None:
self._log_stats.error('Corresponding xpl-stat can not be found for xplstatparam %s' % (statparam))
continue
dev = self._db.get_device(stat.device_id)
if dev is None:
self._log_stats.error('Corresponding device can not be found for xpl-stat %s' % (stat))
continue
# xpl-trig
self.stats.append(self._Stat(self.myxpl, dev, stat, sen, \
"xpl-trig", self._log_stats, \
self._log_stats, self._db, \
self._event_requests))
# xpl-stat
self.stats.append(self._Stat(self.myxpl, dev, stat, sen, \
"xpl-stat", self._log_stats, \
self._log_stats, self._db, \
self._event_requests))
except:
self._log_stats.error("%s" % traceback.format_exc())
class _Stat:
""" This class define a statistic parser and logger instance
Each instance create a Listener and the associated callbacks
"""
def __init__(self, xpl, dev, stat, sensor, xpl_type, log_stats, log_stats_unknown, db, event_requests):
""" Initialize a stat instance
@param xpl : A xpl manager instance
@param dev : A Device reference
@param stat : A XplStat reference
@param sensor: A Sensor reference
@param xpl-type: what xpl-type to listen for
"""
### Rest data
self._event_requests = event_requests
self._db = db
self._log_stats = log_stats
#self._log_stats_unknown = log_stats_unknown
self._dev = dev
self._stat = stat
self._sen = sensor
### build the filter
params = {'schema': stat.schema, 'xpltype': xpl_type}
for p in stat.params:
if p.static:
params[p.key] = p.value
### start the listener
self._log_stats.debug("creating listener for %s" % (params))
self._listener = Listener(self._callback, xpl, params)
def get_listener(self):
""" getter for lsitener object
"""
return self._listener
def _callback(self, message):
""" Callback for the xpl message
@param message : the Xpl message received
"""
my_db = DbHelper()
self._log_stats.debug("Stat received for device %s." \
% (self._dev.name))
current_date = calendar.timegm(time.gmtime())
device_data = []
try:
# find what parameter to store
for p in self._stat.params:
if p.sensor_id is not None:
if p.key in message.data:
value = message.data[p.key]
self._log_stats.debug("Key found %s with value %s." \
% (p.key, value))
store = True
if p.ignore_values:
if value in eval(p.ignore_values):
self._log_stats.debug("Value %s is in the ignore list %s, so not storing." \
% (value, p.ignore_values))
store = False
if store:
# check if we need a conversion
if self._sen.conversion is not None and self._sen.conversion != '':
value = call_package_conversion(\
self._log_stats, self._dev.device_type.plugin_id, \
self._sen.conversion, value)
self._log_stats.debug("Key found %s with value %s after conversion." \
% (p.key, value))
# do the store
device_data.append({"value" : value, "sensor": p.sensor_id})
my_db.add_sensor_history(p.sensor_id, value, current_date)
except:
error = "Error when processing stat : %s" % traceback.format_exc()
print("==== Error in Stats ====")
print(error)
print("========================")
self._log_stats.error(error)
# put data in the event queue
self._event_requests.add_in_queues(self._dev.id,
{"timestamp" : current_date, "device_id" : self._dev.id, "data" : device_data})
del(my_db)
class DBConnector(Plugin, MQRep):
'''
Manage the connection between database and the plugins
Should be the *only* object along with the StatsManager to access to the database on the core side
'''
def __init__(self):
'''
Initialize database and xPL connection
'''
Plugin.__init__(self, 'dbmgr', log_prefix='core_')
# Already done in Plugin
#MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug(u"Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
with self._db.session_scope():
# TODO : move in a function and use it (also used in dbmgr)
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc())
self.log.error(msg)
msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc()))
self.force_leave()
return
self.ready()
# Already done in ready()
#IOLoop.instance().start()
def on_mdp_request(self, msg):
""" Handle Requests over MQ
@param msg : MQ req message
"""
try:
with self._db.session_scope():
# Plugin handles MQ Req/rep also
Plugin.on_mdp_request(self, msg)
# configuration
if msg.get_action() == "config.get":
self._mdp_reply_config_get(msg)
elif msg.get_action() == "config.set":
self._mdp_reply_config_set(msg)
elif msg.get_action() == "config.delete":
self._mdp_reply_config_delete(msg)
# devices list
elif msg.get_action() == "device.get":
self._mdp_reply_devices_result(msg)
# device get params
elif msg.get_action() == "device.params":
self._mdp_reply_devices_params_result(msg)
# device create
elif msg.get_action() == "device.create":
self._mdp_reply_devices_create_result(msg)
# device delete
elif msg.get_action() == "device.delete":
self._mdp_reply_devices_delete_result(msg)
# device update
elif msg.get_action() == "device.update":
self._mdp_reply_devices_update_result(msg)
# deviceparam update
elif msg.get_action() == "deviceparam.update":
self._mdp_reply_deviceparam_update_result(msg)
# sensor update
elif msg.get_action() == "sensor.update":
self._mdp_reply_sensor_update_result(msg)
# sensor history
elif msg.get_action() == "sensor_history.get":
self._mdp_reply_sensor_history(msg)
except:
msg = "Error while processing request. Message is : {0}. Error is : {1}".format(msg, traceback.format_exc())
self.log.error(msg)
def _mdp_reply_config_get(self, data):
""" Reply to config.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] not in ["plugin", "brain", "interface"]:
status = False
reason = "Configuration request not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if 'key' not in msg_data:
get_all_keys = True
key = "*"
else:
get_all_keys = False
key = msg_data['key']
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('key', key) # we let this here to display key or * depending on the case
try:
if get_all_keys == True:
config = self._db.list_plugin_config(type, name, host)
self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, config))
json_config = {}
for elt in config:
json_config[elt.key] = self.convert(elt.value)
msg.add_data('data', json_config)
else:
value = self._fetch_techno_config(name, type, host, key)
# temporary fix : should be done in a better way (on db side)
value = self.convert(value)
self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, value))
msg.add_data('value', value)
except:
status = False
reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(type, name, host, key, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_set(self, data):
""" Reply to config.set MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config set : missing 'type' field : {0}".format(data)
if msg_data['type'] not in ["plugin", "brain", "interface"]:
status = False
reason = "You are not able to configure items for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config set : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config set : missing 'host' field : {0}".format(data)
if 'data' not in msg_data:
status = False
reason = "Config set : missing 'data' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
data = msg_data['data']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
# we add a configured key set to true to tell the UIs and plugins that there are some configuration elements
self._db.set_plugin_config(type, name, host, "configured", True)
for key in msg_data['data']:
self._db.set_plugin_config(type, name, host, key, data[key])
self.publish_config_updated(type, name, host)
except:
reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(type, name, host, traceback.format_exc())
status = False
self.log.error(reason)
msg.add_data('status', status)
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_delete(self, data):
""" Reply to config.delete MQ req
Delete all the config items for the given type, name and host
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] not in ["plugin", "brain", "interface"]:
status = False
reason = "Configuration deletion not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
self._db.del_plugin_config(type, name, host)
self.log.info(u"Delete config for {0} {1}".format(type, name))
self.publish_config_updated(type, name, host)
except:
status = False
reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(type, name, host, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _fetch_techno_config(self, name, type, host, key):
'''
Fetch a plugin global config value in the database
@param name : the plugin of the element
@param host : hostname
@param key : the key of the config tuple to fetch
'''
try:
try:
result = self._db.get_plugin_config(type, name, host, key)
# tricky loop as workaround for a (sqlalchemy?) bug :
# sometimes the given result is for another plugin/key
# so while we don't get the good data, we loop
# This bug happens rarely
while result.id != name or \
result.type != type or \
result.hostname != host or \
result.key != key:
self.log.debug(u"Bad result : {0}-{1}/{2} != {3}/{4}".format(result.id, result.type, result.key, plugin, key))
result = self._db.get_plugin_config(type, name, host, key)
val = result.value
if val == '':
val = "None"
return val
except AttributeError:
# if no result is found
#self.log.error(u"Attribute error : {0}".format(traceback.format_exc()))
return "None"
except:
msg = "No config found host={0}, plugin={1}, key={2}".format(host, name, key)
self.log.warn(msg)
return "None"
def _mdp_reply_devices_delete_result(self, data):
status = True
reason = False
self.log.debug(u"Deleting device : {0}".format(data))
try:
did = data.get_data()['did']
if did:
res = self._db.del_device(did)
if not res:
status = False
else:
status = True
else:
status = False
reason = "There is no such device"
self.log.debug(reason)
# delete done
except DbHelperException as d:
status = False
reason = "Error while deleting device: {0}".format(d.value)
self.log.error(reason)
except:
status = False
reason = "Error while deleting device: {0}".format(traceback.format_exc())
self.log.error(reason)
# send the result
msg = MQMessage()
msg.set_action('device.delete.result')
msg.add_data('status', status)
if reason:
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
# send the pub message
if status and res:
self._pub.send_event('device.update',
{"device_id" : did,
"client_id" : res.client_id})
def _mdp_reply_sensor_update_result(self, data):
status = True
reason = False
self.log.debug(u"Updating sensor : {0}".format(data))
try:
data = data.get_data()
if 'sid' in data:
sid = data['sid']
if 'history_round' not in data:
hround = None
else:
hround = data['history_round']
if 'history_store' not in data:
hstore = None
else:
hstore = data['history_store']
if 'history_max' not in data:
hmax = None
else:
hmax = data['history_max']
if 'history_expire' not in data:
hexpire = None
else:
hexpire = data['history_expire']
if 'timeout' not in data:
timeout = None
else:
timeout = data['timeout']
if 'formula' not in data:
formula = None
else:
formula = data['formula']
if 'data_type' not in data:
data_type = None
else:
data_type = data['data_type']
# do the update
res = self._db.update_sensor(sid, \
history_round=hround, \
history_store=hstore, \
history_max=hmax, \
history_expire=hexpire, \
timeout=timeout, \
formula=formula, \
data_type=data_type)
if not res:
status = False
else:
status = True
else:
status = False
reason = "There is no such sensor"
self.log.debug(reason)
# delete done
except DbHelperException as d:
status = False
reason = "Error while updating sensor: {0}".format(d.value)
self.log.error(reason)
except:
status = False
reason = "Error while updating sensor: {0}".format(traceback.format_exc())
self.log.error(reason)
# send the result
msg = MQMessage()
msg.set_action('sensor.update.result')
msg.add_data('status', status)
if reason:
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
# send the pub message
if status and res:
dev = self._db.get_device(res.device_id)
self._pub.send_event('device.update',
{"device_id" : res.device_id,
"client_id" : dev['client_id']})
def _mdp_reply_deviceparam_update_result(self, data):
status = True
reason = False
self.log.debug(u"Updating device param : {0}".format(data))
try:
data = data.get_data()
if 'dpid' in data:
dpid = data['dpid']
val = data['value']
# do the update
res = self._db.udpate_device_param(dpid, value=val)
if not res:
status = False
else:
status = True
else:
status = False
reason = "There is no such device param"
self.log.debug(reason)
# delete done
except DbHelperException as d:
status = False
reason = "Error while updating device param: {0}".format(d.value)
self.log.error(reason)
except:
status = False
reason = "Error while updating device param: {0}".format(traceback.format_exc())
self.log.error(reason)
# send the result
msg = MQMessage()
msg.set_action('deviceparam.update.result')
msg.add_data('status', status)
if reason:
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
# send the pub message
if status and res:
dev = self._db.get_device(res.device_id)
self._pub.send_event('device.update',
{"device_id" : res.device_id,
"client_id" : dev['client_id']})
def _mdp_reply_devices_update_result(self, data):
status = True
reason = False
self.log.debug(u"Updating device : {0}".format(data))
try:
data = data.get_data()
if 'did' in data:
did = data['did']
if 'name' not in data:
name = None
else:
name = data['name']
if 'reference' not in data:
ref = None
else:
ref = data['reference']
if 'description' not in data:
desc = None
else:
desc = data['description']
# do the update
res = self._db.update_device(did, \
d_name=name, \
d_description=desc, \
d_reference=ref)
if not res:
status = False
else:
status = True
else:
status = False
reason = "There is no such device"
self.log.debug(reason)
# delete done
except DbHelperException as d:
status = False
reason = "Error while updating device: {0}".format(d.value)
self.log.error(reason)
except:
status = False
reason = "Error while updating device: {0}".format(traceback.format_exc())
self.log.error(reason)
# send the result
msg = MQMessage()
msg.set_action('device.update.result')
msg.add_data('status', status)
if reason:
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
# send the pub message
if status and res:
self._pub.send_event('device.update',
{"device_id" : res.id,
"client_id" : res.client_id})
def _mdp_reply_devices_create_result(self, data):
status = True
reason = False
result = False
# get the filled package json
params = data.get_data()['data']
# get the json
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('device_types.get')
msg.add_data('device_type', params['device_type'])
res = cli.request('manager', msg.get(), timeout=10)
del cli
if res is None:
status = False
reason = "Manager is not replying to the mq request"
pjson = res.get_data()
if pjson is None:
status = False
reason = "No data for {0} found by manager".format(params['device_type'])
pjson = pjson[params['device_type']]
if pjson is None:
status = False
reason = "The json for {0} found by manager is empty".format(params['device_type'])
if status:
# call the add device function
res = self._db.add_full_device(params, pjson)
if not res:
status = False
reason = "An error occured while adding the device in database. Please check the file dbmgr.log for more informations"
else:
status = True
reason = False
result = res
msg = MQMessage()
msg.set_action('device.create.result')
if reason:
msg.add_data('reason', reason)
if result:
msg.add_data('result', result)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
# send the pub message
if status and res:
self._pub.send_event('device.update',
{"device_id" : res['id'],
"client_id" : res['client_id']})
def _mdp_reply_devices_params_result(self, data):
"""
Reply to device.params mq req
@param data : MQ req message
=> should contain
- device_type
"""
status = True
try:
# check we have all the needed info
msg_data = data.get_data()
if 'device_type' not in msg_data:
status = False
reason = "Device params request : missing 'cevice_type' field : {0}".format(data)
else:
dev_type_id = msg_data['device_type']
# check the received info
if status:
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('device_types.get')
msg.add_data('device_type', dev_type_id)
res = cli.request('manager', msg.get(), timeout=10)
del cli
if res is None:
status = False
reason = "Manager is not replying to the mq request"
if status:
pjson = res.get_data()
if pjson is None:
status = False
reason = "No data for {0} found by manager".format(msg_data['device_type'])
if status:
pjson = pjson[dev_type_id]
if pjson is None:
status = False
reason = "The json for {0} found by manager is empty".format(msg_data['device_type'])
self.log.debug("Device Params result : json received by the manager is : {0}".format(pjson))
if not status:
# we don't have all info so exit
msg = MQMessage()
msg.set_action('device.params.result')
msg.add_data('result', 'Failed')
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
return
# we have the json now, build the params
msg = MQMessage()
msg.set_action('device.params.result')
stats = []
result = {}
result['device_type'] = dev_type_id
result['name'] = ""
result['reference'] = ""
result['description'] = ""
# append the global xpl and on-xpl params
result['xpl'] = []
result['global'] = []
for param in pjson['device_types'][dev_type_id]['parameters']:
if param['xpl']:
del param['xpl']
result['xpl'].append(param)
else:
del param['xpl']
result['global'].append(param)
# find the xplCommands
result['xpl_commands'] = {}
for cmdn in pjson['device_types'][dev_type_id]['commands']:
cmd = pjson['commands'][cmdn]
if 'xpl_command'in cmd:
xcmdn = cmd['xpl_command']
xcmd = pjson['xpl_commands'][xcmdn]
result['xpl_commands'][xcmdn] = []
stats.append( xcmd['xplstat_name'] )
for param in xcmd['parameters']['device']:
result['xpl_commands'][xcmdn].append(param)
# find the xplStats
sensors = pjson['device_types'][dev_type_id]['sensors']
#print("SENSORS = {0}".format(sensors))
for xstatn in pjson['xpl_stats']:
#print("XSTATN = {0}".format(xstatn))
xstat = pjson['xpl_stats'][xstatn]
for sparam in xstat['parameters']['dynamic']:
#print("XSTATN = {0}, SPARAM = {1}".format(xstatn, sparam))
#if 'sensor' in sparam and xstatn in sensors:
# => This condition was used to fix a bug which occurs while creating complexe devices for rfxcom
# But is introduced a bug for the geoloc plugin...
# In fact we had to fix the rfxcom info.json file (open_close uses now rssi_open_close instead of
# rssi_lighting2
# So, this one is NOT the good one.
if 'sensor' in sparam:
# => this condition was the original one restored to make the geoloc pluin ok for tests
# Strangely, there is no issue while using the admin (which uses only mq)
# but is sucks with test library which uses rest...
# This one is the good one
if sparam['sensor'] in sensors:
#print("ADD")
stats.append(xstatn)
result['xpl_stats'] = {}
#print("STATS = {0}".format(stats))
for xstatn in stats:
xstat = pjson['xpl_stats'][xstatn]
result['xpl_stats'][xstatn] = []
for param in xstat['parameters']['device']:
result['xpl_stats'][xstatn].append(param)
# return the data
msg.add_data('result', result)
self.log.debug(msg.get())
self.reply(msg.get())
except:
self.log.error("Error when replying to device.params for data={0}. Error: {1}".format(data, traceback.format_exc()))
def _mdp_reply_devices_result(self, data):
""" Reply to device.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('device.result')
status = True
msg_data = data.get_data()
# request for all devices
if 'type' not in msg_data and \
'name' not in msg_data and \
'host' not in msg_data and \
'timestamp' not in msg_data:
reason = ""
status = True
dev_list = self._db.list_devices()
dev_json = dev_list
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('devices', dev_json)
elif 'timestamp'in msg_data:
# request for all devices that changed after timestamp
reason = ""
status = True
dev_list = self._db.list_devices_by_timestamp(msg_data['timestamp'])
dev_json = dev_list
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('devices', dev_json)
else:
# request for all devices of one client
if 'type' not in msg_data:
status = False
reason = "Devices request : missing 'type' field : {0}".format(data)
if 'name' not in msg_data:
status = False
reason = "Devices request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Devices request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(type, name, host))
#dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False),
dev_json = dev_list
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('devices', dev_json)
self.reply(msg.get())
def _mdp_reply_sensor_history(self, data):
""" Reply to sensor_history.get MQ req
@param data : MQ req message
If no other param than the sensor id, return the last value
"""
msg = MQMessage()
msg.set_action('sensor_history.result')
status = True
reason = ""
### process parameters
msg_data = data.get_data()
if 'sensor_id' in msg_data:
sensor_id = msg_data['sensor_id']
else:
reason = "ERROR when getting sensor history. No sensor_id declared in the message"
self.log.error(reason)
status = False
sensor_id = None
if 'mode' in msg_data:
if msg_data['mode'] == "last":
mode = "last"
elif msg_data['mode'] == "period":
mode = "period"
else:
reason = "ERROR when getting sensor history. No valid type (last, from) declared in the message"
self.log.error(reason)
status = False
mode = None
else:
reason = "ERROR when getting sensor history. No type (last, from) declared in the message"
self.log.error(reason)
status = False
sensor_id = None
values = None
### last N values
if mode == "last":
if 'number' in msg_data:
number = msg_data['number']
else:
number = 1
try:
history = self._db.list_sensor_history(sensor_id, number)
if len(history) == 0:
#values = None
values = self._db.get_last_sensor_value(sensor_id)
else:
values = self._db.list_sensor_history(sensor_id, number)
except:
self.log.error("ERROR when getting sensor history for id = {0} : {1}".format(sensor_id, traceback.format_exc()))
reason = "ERROR : {0}".format(traceback.format_exc())
status = False
### period
elif mode == "period":
if 'from' in msg_data:
frm = msg_data['from']
else:
reason = "ERROR when getting sensor history. No key 'from' defined for mode = 'period'!"
self.log.error(reason)
status = False
frm = None
if 'to' in msg_data:
to = msg_data['to']
else:
to = None
if frm != None and to == None:
values = self._db.list_sensor_history_between(sensor_id, frm)
print(values)
else:
# TODO
values = "TODO"
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('sensor_id', sensor_id)
msg.add_data('mode', mode)
msg.add_data('values', values)
self.reply(msg.get())
def convert(self, data):
""" Do some conversions on data
"""
if data == "True":
data = True
if data == "False":
data = False
return data
def publish_config_updated(self, type, name, host):
""" Publish over the MQ a message to inform that a plugin configuration has been updated
@param type : package type (plugin)
@param name : package name
@param host : host
"""
self.log.debug("Publish configuration update notification for {0}-{1}.{2}".format(type, name, host))
self._pub.send_event('plugin.configuration',
{"type" : type,
"name" : name,
"host" : host,
"event" : "updated"})
def check_args(argv):
"""Check arguments passed to the program"""
def usage(prog_name):
"""Print program usage"""
print("Usage : %s [-s [all|minute[,hour][,day][,week][,month[,year]]] [-I]" % prog_name)
print("-s, --statistics=STATS_LIST\tSTATS_LIST can be : all or minute,hours,day,week,month,year")
print("-I, --noinsert\t\t\tUse existing data in the database")
if __name__ == "__main__":
stats_filter = False
try:
opts, args = getopt.getopt(sys.argv[1:], "hs:I", ["help", "stats=", "noinsert"])
_db = DbHelper(use_test_db=True)
print("Using %s database" % _db.get_db_type())
for opt, arg_list in opts:
if opt in ("-s", "--stats"):
possible_args = ['all', 'minute', 'hour', 'day', 'week', 'month', 'year']
if arg_list is not None:
filter_list = arg_list.split(",")
# Check args are ok for -s option
for p_filter in filter_list:
if p_filter not in filter_list:
print("Wrong argument for statistics, must be one of : %s" % (",".join(possible_args)))
usage(sys.argv[0])
if 'all' in filter_list:
filter_list = possible_args[1:]
stats_filter = True
class DBConnector(Plugin, MQRep):
'''
Manage the connection between database and the plugins
Should be the *only* object along with the StatsManager to access to the database on the core side
'''
def __init__(self):
'''
Initialize database and xPL connection
'''
Plugin.__init__(self, 'dbmgr')
# Already done in Plugin
#MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug(u"Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
with self._db.session_scope():
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}".format(nb_test, DATABASE_CONNECTION_NUM_TRY)
self.log.error(msg)
msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc()))
self.force_leave()
return
self.ready()
# Already done in ready()
#IOLoop.instance().start()
def on_mdp_request(self, msg):
""" Handle Requests over MQ
@param msg : MQ req message
"""
try:
with self._db.session_scope():
# Plugin handles MQ Req/rep also
Plugin.on_mdp_request(self, msg)
# configuration
if msg.get_action() == "config.get":
self._mdp_reply_config_get(msg)
elif msg.get_action() == "config.set":
self._mdp_reply_config_set(msg)
elif msg.get_action() == "config.delete":
self._mdp_reply_config_delete(msg)
# devices list
elif msg.get_action() == "device.get":
self._mdp_reply_devices_result(msg)
# device get params
elif msg.get_action() == "device.params":
self._mdp_reply_devices_params_result(msg)
# device create
elif msg.get_action() == "device.create":
self._mdp_reply_devices_create_result(msg)
# device delete
elif msg.get_action() == "device.delete":
self._mdp_reply_devices_delete_result(msg)
except:
msg = "Error while processing request. Message is : {0}. Error is : {1}".format(msg, traceback.format_exc())
self.log.error(msg)
def _mdp_reply_config_get(self, data):
""" Reply to config.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config request not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if 'key' not in msg_data:
get_all_keys = True
key = "*"
else:
get_all_keys = False
key = msg_data['key']
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('key', key) # we let this here to display key or * depending on the case
try:
if get_all_keys == True:
config = self._db.list_plugin_config(name, host)
self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, config))
json_config = {}
for elt in config:
json_config[elt.key] = self.convert(elt.value)
msg.add_data('data', json_config)
else:
value = self._fetch_techno_config(name, host, key)
# temporary fix : should be done in a better way (on db side)
value = self.convert(value)
self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, value))
msg.add_data('value', value)
except:
status = False
reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(type, name, host, key, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_set(self, data):
""" Reply to config.set MQ req
@param data : MQ req message
"""
print "#################"
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config set : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config set not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config set : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config set : missing 'host' field : {0}".format(data)
if 'data' not in msg_data:
status = False
reason = "Config set : missing 'data' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
data = msg_data['data']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
# we add a configured key set to true to tell the UIs and plugins that there are some configuration elements
self._db.set_plugin_config(name, host, "configured", True)
for key in msg_data['data']:
self._db.set_plugin_config(name, host, key, data[key])
self.publish_config_updated(type, name, host)
except:
reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(type, name, host, traceback.format_exc())
status = False
self.log.error(reason)
msg.add_data('status', status)
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_delete(self, data):
""" Reply to config.delete MQ req
Delete all the config items for the given type, name and host
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config request not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
self._db.del_plugin_config(name, host)
self.log.info(u"Delete config for {0} {1}".format(type, name))
self.publish_config_updated(type, name, host)
except:
status = False
reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(type, name, host, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _fetch_techno_config(self, name, host, key):
'''
Fetch a plugin global config value in the database
@param name : the plugin of the element
@param host : hostname
@param key : the key of the config tuple to fetch
'''
try:
try:
result = self._db.get_plugin_config(name, host, key)
# tricky loop as workaround for a (sqlalchemy?) bug :
# sometimes the given result is for another plugin/key
# so while we don't get the good data, we loop
# This bug happens rarely
while result.id != name or \
result.hostname != host or \
result.key != key:
self.log.debug(u"Bad result : {0}/{1} != {2}/{3}".format(result.id, result.key, plugin, key))
result = self._db.get_plugin_config(name, host, key)
val = result.value
if val == '':
val = "None"
return val
except AttributeError:
# if no result is found
#self.log.error(u"Attribute error : {0}".format(traceback.format_exc()))
return "None"
except:
msg = "No config found host={0}, plugin={1}, key={2}".format(host, name, key)
self.log.warn(msg)
return "None"
def _mdp_reply_devices_delete_result(self, data):
status = True
reason = False
try:
did = data.get_data()['did']
if did:
res = self._db.del_device(did)
if not res:
status = False
else:
status = True
else:
status = False
reason = "Device delete failed"
# delete done
self.reload_stats()
except DbHelperException as d:
status = False
reason = "Error while deleting device: {0}".format(d.value)
except:
status = False
reason = "Error while deleting device: {0}".format(traceback.format_exc())
# send the result
msg = MQMessage()
msg.set_action('device.delete.result')
msg.add_data('status', status)
if reason:
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_devices_create_result(self, data):
status = True
reason = False
result = False
# get the filled package json
params = data.get_data()['data']
# get the json
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('device_types.get')
msg.add_data('device_type', params['device_type'])
res = cli.request('manager', msg.get(), timeout=10)
del cli
if res is None:
status = False
reason = "Manager is not replying to the mq request"
pjson = res.get_data()
if pjson is None:
status = False
reason = "No data for {0} found by manager".format(params['device_type'])
pjson = pjson[params['device_type']]
if pjson is None:
status = False
reason = "The json for {0} found by manager is empty".format(params['device_type'])
if status:
# call the add device function
res = self._db.add_full_device(params, pjson)
if not res:
status = False
reason = "An error occured while adding the device in database. Please check the file dbmgr.log for more informations"
else:
status = True
reason = False
result = res
msg = MQMessage()
msg.set_action('device.create.result')
if reason:
msg.add_data('reason', reason)
if result:
msg.add_data('result', result)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_devices_params_result(self, data):
"""
Reply to device.params mq req
@param data : MQ req message
=> should contain
- device_type
"""
status = True
# check we have all the needed info
msg_data = data.get_data()
if 'device_type' not in msg_data:
status = False
reason = "Device params request : missing 'cevice_type' field : {0}".format(data)
else:
dev_type_id = msg_data['device_type']
# check the received info
if status:
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('device_types.get')
msg.add_data('device_type', dev_type_id)
res = cli.request('manager', msg.get(), timeout=10)
del cli
if res is None:
status = False
reason = "Manager is not replying to the mq request"
pjson = res.get_data()
if pjson is None:
status = False
reason = "No data for {0} found by manager".format(msg_data['device_type'])
pjson = pjson[dev_type_id]
if pjson is None:
status = False
reason = "The json for {0} found by manager is empty".format(msg_data['device_type'])
# we have the json now, build the params
msg = MQMessage()
msg.set_action('device.params.result')
stats = []
result = {}
result['device_type'] = dev_type_id
result['name'] = ""
result['reference'] = ""
result['description'] = ""
# append the global xpl and on-xpl params
result['xpl'] = []
result['global'] = []
for param in pjson['device_types'][dev_type_id]['parameters']:
if param['xpl']:
del param['xpl']
result['xpl'].append(param)
else:
del param['xpl']
result['global'].append(param)
# find the xplCommands
result['xpl_commands'] = {}
for cmdn in pjson['device_types'][dev_type_id]['commands']:
cmd = pjson['commands'][cmdn]
if 'xpl_command'in cmd:
xcmdn = cmd['xpl_command']
xcmd = pjson['xpl_commands'][xcmdn]
result['xpl_commands'][xcmdn] = []
stats.append( xcmd['xplstat_name'] )
for param in xcmd['parameters']['device']:
result['xpl_commands'][xcmdn].append(param)
# find the xplStats
sensors = pjson['device_types'][dev_type_id]['sensors']
for xstatn in pjson['xpl_stats']:
xstat = pjson['xpl_stats'][xstatn]
if xstat['parameters']['dynamic'] in sensors:
stats.append(xstatn)
result['xpl_stats'] = {}
for xstatn in stats:
xtat = pjson['xpl_stats'][xstatn]
result['xpl_stats'][xstatn] = []
for param in xstat['parameters']['device']:
result['xpl_stats'][xstatn].append(param)
# return the data
msg.add_data('result', result)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_devices_result(self, data):
""" Reply to device.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('device.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Devices request : missing 'type' field : {0}".format(data)
if 'name' not in msg_data:
status = False
reason = "Devices request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Devices request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
#if type == "plugin":
# type = DMG_VENDOR_ID
name = msg_data['name']
host = msg_data['host']
dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(type, name, host))
#dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False),
dev_json = dev_list
print(dev_json)
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('devices', dev_json)
self.reply(msg.get())
def convert(self, data):
""" Do some conversions on data
"""
if data == "True":
data = True
if data == "False":
data = False
return data
def publish_config_updated(self, type, name, host):
""" Publish over the MQ a message to inform that a plugin configuration has been updated
@param type : package type (plugin)
@param name : package name
@param host : host
"""
self.log.debug("Publish configuration update notification for {0}-{1}.{2}".format(type, name, host))
self._pub.send_event('plugin.configuration',
{"type" : type,
"name" : name,
"host" : host,
"event" : "updated"})
def reload_stats(self):
self.log.debug(u"=============== reload stats")
req = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action( 'reload' )
resp = req.request('xplgw', msg.get(), 100)
self.log.debug(u"Reply from xplgw: {0}".format(resp))
self.log.debug(u"=============== reload stats END")
def test_user_config_file(user_home, user_entry):
info("Check user config file contents")
import ConfigParser
config = ConfigParser.ConfigParser()
config.read("/etc/domogik/domogik.cfg")
#check [domogik] section
dmg = dict(config.items('domogik'))
database = dict(config.items('database'))
rest = dict(config.items('rest'))
ok("Config file correctly loaded")
info("Parse [domogik] section")
import domogik
#Check ix xpl port is not used
_check_port_availability("0.0.0.0", 3865, udp=True)
ok("xPL hub IP/port is not bound by anything else")
parent_conn, child_conn = Pipe()
p = Process(target=_test_user_can_write,
args=(
child_conn,
dmg['log_dir_path'],
user_entry,
))
p.start()
p.join()
assert parent_conn.recv(
), "The directory %s for log does not exist or does not have right permissions" % dmg[
'log_dir_path']
assert dmg['log_level'] in [
'debug', 'info', 'warning', 'error', 'critical'
], "The log_level parameter does not have a good value. Must \
be one of debug,info,warning,error,critical"
### obsolete
#if not os.path.isdir(dmg['src_prefix'] + '/share/domogik'):
# try:
# f = os.listdir("%s/share/domogik" % dmg['src_prefix'])
# f.close()
# except OSError:
# fail("Can't access %s/share/domogik. Check %s is available for domogik user (if you are in development mode, be sure the directory which contains the sources is available for domogik user)." % (dmg['src_prefix'],dmg['src_prefix']))
# exit()
ok("[domogik] section seems good")
# check [database] section
info("Parse [database] section")
assert database[
'type'] == 'mysql', "Only mysql database type is supported at the moment"
uid = user_entry.pw_uid
os.setreuid(0, uid)
old_home = os.environ['HOME']
os.environ['HOME'] = user_home
from domogik.common.database import DbHelper
d = DbHelper()
os.setreuid(0, 0)
os.environ['HOME'] = old_home
assert d.get_engine(
) != None, "Engine is not set, it seems something went wrong during connection to the database"
ok("[database] section seems good")
# Check [rest] section
info("Parse [rest] section")
for ipadd in get_ip_for_interfaces(rest['interfaces'].split(",")):
_check_port_availability(ipadd, rest['port'])
ok("Rest server IP/port is not bound by anything else")
class XplManager(XplPlugin, MQAsyncSub):
""" Statistics manager
"""
def __init__(self):
""" Initiate DbHelper, Logs and config
"""
XplPlugin.__init__(self, 'xplgw', log_prefix="")
MQAsyncSub.__init__(\
self, self.zmq, 'xplgw', \
['client.conversion', 'client.list'])
self.log.info(u"XPL manager initialisation...")
self._db = DbHelper()
self.pub = MQPub(zmq.Context(), 'xplgw')
# some initial data sets
self.client_xpl_map = {}
self.client_conversion_map = {}
self._db_sensors = {}
self._db_xplstats = {}
# queue to store the message that needs to be ahndled for sensor checking
self._sensor_queue = Queue.Queue()
# all command handling params
# _lock => to be sure to be thread safe
# _dict => uuid to xplstat translationg
# _pkt => received messages to check
self._cmd_lock_d = threading.Lock()
self._cmd_dict = {}
self._cmd_lock_p = threading.Lock()
self._cmd_pkt = {}
# load some initial data from manager and db
self._load_client_to_xpl_target()
self._load_conversions()
# create a general listener
self._create_xpl_trigger()
# start handling the xplmessages
self._s_thread = self._SensorThread(\
self.log, self._sensor_queue, \
self.client_conversion_map, self.pub)
self._s_thread.start()
# start handling the command reponses in a thread
self._c_thread = self._CommandThread(\
self.log, self._db, self._cmd_lock_d, \
self._cmd_lock_p, self._cmd_dict, self._cmd_pkt, self.pub)
self._c_thread.start()
# start the sensorthread
self.ready()
def on_mdp_request(self, msg):
""" Method called when an mq request comes in
XplPlugin also needs this info, so we need to do a passthrough
"""
try:
XplPlugin.on_mdp_request(self, msg)
if msg.get_action() == "test":
pass
if msg.get_action() == "cmd.send":
self._send_xpl_command(msg)
except Exception as exp:
self.log.error(traceback.format_exc())
def on_message(self, msgid, content):
""" Method called on a subscribed message
"""
try:
if msgid == 'client.conversion':
self._parse_conversions(content)
elif msgid == 'client.list':
self._parse_xpl_target(content)
except Exception as exp:
self.log.error(traceback.format_exc())
def _load_client_to_xpl_target(self):
""" Request the client conversion info
This is an mq req to manager
"""
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('client.list.get')
response = cli.request('manager', msg.get(), timeout=10)
if response:
self._parse_xpl_target(response.get_data())
else:
self.log.error(\
u"Updating client list failed, no response from manager")
def _parse_xpl_target(self, data):
""" Translate the mq data info a dict
for the xpl targets
"""
tmp = {}
for cli in data:
tmp[cli] = data[cli]['xpl_source']
self.client_xpl_map = tmp
def _load_conversions(self):
""" Request the client conversion info
This is an mq req to manager
"""
cli = MQSyncReq(self.zmq)
msg = MQMessage()
msg.set_action('client.conversion.get')
response = cli.request('manager', msg.get(), timeout=10)
if response:
self._parse_conversions(response.get_data())
else:
self.log.error(\
u"Updating conversion list failed, no response from manager")
def _parse_conversions(self, data):
""" Translate the mq data into a dict
"""
tmp = {}
for cli in data:
tmp[cli] = data[cli]
self.client_conversion_map = tmp
def _send_xpl_command(self, data):
""" Reply to config.get MQ req
@param data : MQ req message
Needed info in data:
- cmdid => command id to send
- cmdparams => key/value pair of all params needed for this command
"""
with self._db.session_scope():
self.log.info(u"Received new cmd request: {0}".format(data))
failed = False
request = data.get_data()
if 'cmdid' not in request:
failed = "cmdid not in message data"
if 'cmdparams' not in request:
failed = "cmdparams not in message data"
if not failed:
# get the command
cmd = self._db.get_command(request['cmdid'])
if cmd is not None:
if cmd.xpl_command is not None:
xplcmd = cmd.xpl_command
xplstat = self._db.get_xpl_stat(xplcmd.stat_id)
if xplstat is not None:
# get the device from the db
dev = self._db.get_device(int(cmd.device_id))
msg = XplMessage()
if not dev['client_id'] in self.client_xpl_map.keys():
self._load_client_to_xpl_target()
if not dev['client_id'] in self.client_xpl_map.keys():
failed = "Can not fincd xpl source for {0} client_id".format(dev['client_id'])
else:
msg.set_target(self.client_xpl_map[dev['client_id']])
msg.set_source(self.myxpl.get_source())
msg.set_type("xpl-cmnd")
msg.set_schema(xplcmd.schema)
# static paramsw
for par in xplcmd.params:
msg.add_data({par.key : par.value})
# dynamic params
for par in cmd.params:
if par.key in request['cmdparams']:
value = request['cmdparams'][par.key]
# chieck if we need a conversion
if par.conversion is not None and par.conversion != '':
if dev['client_id'] in self.client_conversion_map:
if par.conversion in self.client_conversion_map[dev['client_id']]:
exec(self.client_conversion_map[dev['client_id']][par.conversion])
value = locals()[par.conversion](value)
msg.add_data({par.key : value})
else:
failed = "Parameter ({0}) for device command msg is not provided in the mq message".format(par.key)
if not failed:
# send out the msg
self.log.debug(u"Sending xplmessage: {0}".format(msg))
self.myxpl.send(msg)
xplstat = self._db.detach(xplstat)
# generate an uuid for the matching answer published messages
if xplstat != None:
resp_uuid = uuid4()
self._cmd_lock_d.acquire()
self._cmd_dict[str(resp_uuid)] = xplstat
self._cmd_lock_d.release()
else:
resp_uuid = None
# send the response
reply_msg = MQMessage()
reply_msg.set_action('cmd.send.result')
reply_msg.add_data('uuid', str(resp_uuid))
reply_msg.add_data('status', True)
reply_msg.add_data('reason', None)
self.log.debug(u"mq reply (success) : {0}".format(reply_msg.get()))
self.reply(reply_msg.get())
if failed:
self.log.error(failed)
reply_msg = MQMessage()
reply_msg.set_action('cmd.send.result')
reply_msg.add_data('uuid', None)
reply_msg.add_data('status', False)
reply_msg.add_data('reason', failed)
self.log.debug(u"mq reply (failed) : {0}".format(reply_msg.get()))
self.reply(reply_msg.get())
def _create_xpl_trigger(self):
""" Create a listener to catch
all xpl-stats and xpl-trig messages
"""
Listener(self._xpl_callback, self.myxpl, {'xpltype': 'xpl-trig'})
Listener(self._xpl_callback, self.myxpl, {'xpltype': 'xpl-stat'})
def _xpl_callback(self, pkt):
""" The callback for the xpl messages
push them into the needed queues
"""
item = {}
item["msg"] = pkt
item["clientId"] = next((cli for cli, xpl in self.client_xpl_map.items() if xpl == pkt.source), None)
self._sensor_queue.put(item)
self.log.debug(u"Adding new message to the sensorQueue, current length = {0}".format(self._sensor_queue.qsize()))
#self.log.debug(u"Adding new message to the sensorQueue, current length = {0}, message = {1}".format(self._sensor_queue.qsize(), pkt))
self._cmd_lock_p.acquire()
# only do this when we have outstanding commands
if len(self._cmd_dict) > 0:
self._cmd_pkt[time.time()] = pkt
self.log.debug(u"Adding new message to the cmdQueue, current length = {0}".format(len(self._cmd_dict)))
#self.log.debug(u"Adding new message to the cmdQueue, current length = {0}, message = {1}".format(len(self._cmd_dict), pkt))
self._cmd_lock_p.release()
class _CommandThread(threading.Thread):
""" commandthread class
Class responsible for handling one xpl command
"""
def __init__(self, log, db, lock_d, lock_p, dic, pkt, pub):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._lock_d = lock_d
self._lock_p = lock_p
self._dict = dic
self._pkt = pkt
self._pub = pub
def run(self):
while True:
# remove old pkts
self._lock_p.acquire()
for pkt in self._pkt.keys():
if pkt < time.time() - CMDTIMEOUT:
self._log.warning(u"Delete packet too old (timeout reached) : {0}".format(pkt))
del(self._pkt[pkt])
self._lock_p.release()
# now try to match if we have enough data
if len(self._dict) > 0 and len(self._pkt) > 0:
todel_pkt = []
todel_dict = []
for uuid, search in self._dict.items():
for tim, pkt in self._pkt.items():
if search.schema == pkt.schema:
found = True
for par in search.params:
if par.key not in pkt.data:
if par.value != pkt.data[par.key]:
found = False
elif par.multiple is not None and len(par.multiple) == 1:
if pkt.data[par.key] not in par.value.split(par.multiple):
found = False
if found:
self._log.info(u"Found response message to command with uuid: {0}".format(uuid))
# publish the result
self._pub.send_event('command.result', \
{"uuid" : uuid})
todel_pkt.append(tim)
todel_dict.append(uuid)
# now go and delete the unneeded data
self._lock_p.acquire()
for tim in todel_pkt:
if tim in self._pkt:
del(self._pkt[tim])
#self._log.debug(u"Deleting message from the cmdQueue, current length = {0}".format(len(self._pkt)))
# TODO : remove or comment the 2 following lines
#self._log.debug(u"Data to delete : {0}".format(todel_dict))
#self._log.debug(u"Content before deletion : {0}".format(self._dict))
self._lock_p.release()
self._lock_d.acquire()
for tim in todel_dict:
if tim in self._dict:
del(self._dict[tim])
self._lock_d.release()
todel_pkt = []
todel_dict = []
else:
# nothing todo, sleep a second
time.sleep(1)
class _SensorThread(threading.Thread):
""" SensorThread class
Class that will handle the sensor storage in a seperated thread
This will get messages from the SensorQueue
"""
def __init__(self, log, queue, conv, pub):
threading.Thread.__init__(self)
self._db = DbHelper()
self._log = log
self._queue = queue
self._conv = conv
self._pub = pub
def _find_storeparam(self, item):
#print("ITEM = {0}".format(item['msg']))
found = False
tostore = []
for xplstat in self._db.get_all_xpl_stat():
sensors = 0
matching = 0
statics = 0
if xplstat.schema == item["msg"].schema:
#print(" XPLSTAT = {0}".format(xplstat))
# we found a possible xplstat
# try to match all params and try to find a sensorid and a vlaue to store
for param in xplstat.params:
#print(" PARAM = {0}".format(param))
### Caution !
# in case you, who are reading this, have to debug something like that :
# 2015-08-16 22:04:26,190 domogik-xplgw INFO Storing stat for device 'Garage' (6) and sensor 'Humidity' (69): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,306 domogik-xplgw INFO Storing stat for device 'Salon' (10) and sensor 'Humidity' (76): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,420 domogik-xplgw INFO Storing stat for device 'Chambre d'Ewan' (11) and sensor 'Humidity' (80): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,533 domogik-xplgw INFO Storing stat for device 'Chambre des parents' (12) and sensor 'Humidity' (84): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,651 domogik-xplgw INFO Storing stat for device 'Chambre de Laly' (13) and sensor 'Humidity' (88): key 'current' with value '53' after conversion.
# 2015-08-16 22:04:26,770 domogik-xplgw INFO Storing stat for device 'Entrée' (17) and sensor 'Humidity' (133): key 'current' with value '53' after conversion.
#
# which means that for a single xPL message, the value is stored in several sensors (WTF!!! ?)
# It can be related to the fact that the device address key is no more corresponding between the plugin (info.json and xpl sent by python) and the way the device was create in the databse
# this should not happen, but in case... well, we may try to find a fix...
if param.key in item["msg"].data and param.static:
statics = statics + 1
if param.multiple is not None and len(param.multiple) == 1 and item["msg"].data[param.key] in param.value.split(param.multiple):
matching = matching + 1
elif item["msg"].data[param.key] == param.value:
matching = matching + 1
# now we have a matching xplstat, go and find all sensors
if matching == statics:
#print(" MATHING !!!!!")
for param in xplstat.params:
if param.key in item["msg"].data and not param.static:
#print(" ===> TOSTORE !!!!!!!!! : {0}".format({'param': param, 'value': item["msg"].data[param.key]}))
tostore.append( {'param': param, 'value': item["msg"].data[param.key]} )
if len(tostore) > 0:
found = True
if found:
return (found, tostore)
else:
return False
def run(self):
while True:
try:
item = self._queue.get()
self._log.debug(u"Getting item from the sensorQueue, current length = {0}".format(self._queue.qsize()))
# if clientid is none, we don't know this sender so ignore
# TODO check temp disabled until external members are working
#if item["clientId"] is not None:
if True:
with self._db.session_scope():
fdata = self._find_storeparam(item)
if fdata:
#// ICI !!!
self._log.debug(u"Found a matching sensor, so starting the storage procedure. Sensor : {0}".format(fdata))
for data in fdata[1]:
value = data['value']
storeparam = data['param']
current_date = calendar.timegm(time.gmtime())
store = True
if storeparam.ignore_values:
if value in eval(storeparam.ignore_values):
self._log.debug(u"Value {0} is in the ignore list {0}, so not storing.".format(value, storeparam.ignore_values))
store = False
if store:
# get the sensor and dev
sen = self._db.get_sensor(storeparam.sensor_id)
dev = self._db.get_device(sen.device_id)
# check if we need a conversion
if sen.conversion is not None and sen.conversion != '':
if dev['client_id'] in self._conv and sen.conversion in self._conv[dev['client_id']]:
self._log.debug( \
u"Calling conversion {0}".format(sen.conversion))
exec(self._conv[dev['client_id']][sen.conversion])
value = locals()[sen.conversion](value)
self._log.info( \
u"Storing stat for device '{0}' ({1}) and sensor '{2}' ({3}): key '{4}' with value '{5}' after conversion." \
.format(dev['name'], dev['id'], sen.name, sen.id, storeparam.key, value))
# do the store
try:
self._db.add_sensor_history(\
storeparam.sensor_id, \
value, \
current_date)
except Exception as exp:
self._log.error(u"Error when adding sensor history : {0}".format(traceback.format_exc()))
else:
self._log.debug(u"Don't need to store this value")
# publish the result
self._pub.send_event('device-stats', \
{"timestamp" : current_date, \
"device_id" : dev['id'], \
"sensor_id" : sen.id, \
"stored_value" : value})
except Queue.Empty:
# nothing in the queue, sleep for 1 second
time.sleep(1)
except Exception as exp:
self._log.error(traceback.format_exc())
ret = {}
for d in devs:
d = list(d)
if d[0] not in ret:
ret[d[0]] = {}
ret[d[0]]['name'] = d[1]
ret[d[0]]['keys'] = []
ret[d[0]]['keys'].append( d[2] )
else:
ret[d[0]]['keys'].append( d[2] )
return ret
if __name__ == "__main__":
# 0- connect to the DB
db = DbHelper()
db.open_session()
# 1- list current devices (odl ones)
do = True
old_devs = corellateOld(db.upgrade_list_old())
while do:
print old_devs
# show dev menu
i = 1
for dev in old_devs:
print("{0}. {1}".format(i, old_devs[dev]['name']))
i = i + 1
print("0. Exit")
sel = i + 1
while sel > i:
class DBConnector(XplPlugin, MQRep):
'''
Manage the connection between database and the xPL stuff
Should be the *only* object along with the StatsManager to access to the database on the core side
'''
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug("Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (nb_test, DATABASE_CONNECTION_NUM_TRY)
print(msg)
self.log.error(msg)
msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT
print(msg)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
print(msg)
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
print(msg)
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error("Error while starting database engine : %s" % traceback.format_exc())
self.force_leave()
return
Listener(self._request_config_cb, self.myxpl, {'schema': 'domogik.config', 'xpltype': 'xpl-cmnd'})
self.enable_hbeat()
IOLoop.instance().start()
def on_mdp_request(self, msg):
if msg.get_action() == "config.get":
plugin = msg._data['plugin']
hostname = msg._data['hostname']
key = msg._data['key']
if 'element' in msg._data.keys():
element = msg._data['element']
else:
element = None
if element:
self._mdp_reply(plugin, hostname, key, self._fetch_elmt_config(plugin, element, key), element)
elif not key:
print 'TODO'
else:
self._mdp_reply(plugin, hostname, key, self._fetch_techno_config(plugin, hostname, key))
elif msg._action == "config.set":
print 'TODO'
def _mdp_reply(self, plugin, hostname, key, value, element=None):
msg = MQMessage()
msg.setaction( 'config.result' )
msg.add_data('plugin', plugin)
msg.add_data('hostname', hostname)
msg.add_data('key', key)
msg.add_data('value', value)
if element:
msg.add_data('element', element)
print msg.get()
self.reply(msg.get())
def _request_config_cb(self, message):
'''
Callback to receive a request for some config stuff
@param message : the xPL message
'''
#try:
self._db = DbHelper(engine=self._engine)
techno = message.data['plugin']
hostname = message.data['hostname']
key = message.data['key']
msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if "value" in message.data:
new_value = message.data['value']
else:
new_value = None
if "element" in message.data:
element = message.data['element']
else:
element = None
msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
# Set configuration
if new_value:
msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, new_value)
print msg
self.log.debug(msg)
self._set_config(techno, hostname, key, new_value)
# Send configuration
else:
msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if element:
msg = "Request h=%s, t=%s, k=%s (send if element)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(techno, hostname, key, self._fetch_elmt_config(techno, element, key), element)
else:
msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
if not key:
msg = "Request h=%s, t=%s, k=%s (send if not key)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
keys = self._fetch_techno_config(techno, hostname, key).keys()
values = self._fetch_techno_config(techno, hostname, key).values()
self._send_config(techno, hostname, keys, values)
else:
msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % (hostname, techno, key)
print(msg)
self.log.debug(msg)
self._send_config(techno, hostname, key, self._fetch_techno_config(techno, hostname, key))
def _send_config(self, plugin, hostname, key, value, element = None):
'''
Send a config value message for an element's config item
@param plugin : the plugin of the element
@param hostname : hostname
@param element : the name of the element
@param key : the key or list of keys of the config tuple(s) to fetch
@param value : the value or list of values corresponding to the key(s)
'''
msg = "Response h=%s, t=%s, k=%s, v=%s" % (hostname, plugin, key, value)
print(msg)
self.log.debug(msg)
mess = XplMessage()
mess.set_type('xpl-stat')
mess.set_schema('domogik.config')
mess.add_data({'plugin' : plugin})
mess.add_data({'hostname' : hostname})
if element:
mess.add_data({'element' : element})
# If key/value are lists, then we add a key=value for each item
if isinstance(key, list):
for (_key, _val) in zip(key, value):
mess.add_data({_key : _val})
else:
mess.add_data({key : value})
# mess.set_conf_key('target', plugin)
self.myxpl.send(mess)
def _fetch_elmt_config(self, techno, element, key):
'''
Fetch an element's config value in the database
@param techno : the plugin of the element
@param element : the name of the element
@param key : the key of the config tuple to fetch
'''
#TODO : use the database
vals = {'x10': {'a3': {},
'a2': {},
}
}
return vals[techno][element][key]
def _fetch_techno_config(self, techno, hostname, key):
'''
Fetch a plugin global config value in the database
@param techno : the plugin of the element
@param hostname : hostname
@param key : the key of the config tuple to fetch
'''
# This array is here for information only but is not used anymore
# Values are now on the database
#vals = {'x10': {'heyu-cfg-path':'/etc/heyu/x10.conf',
# 'heyu-file-0': 'TTY /dev/ttyUSB0',
# 'heyu-file-1': 'TTY_AUX /dev/ttyUSB0 RFXCOM',
# 'heyu-file-2': 'ALIAS back_door D5 DS10A 0x677'},
# 'global': {'pid-dir-path': '/var/run/'},
# 'onewire': {'temperature_refresh_delay' : '10'},
# 'cidmodem': {'device' : '/dev/ttyUSB1',
# 'nbmaxtry' : '10',
# 'interval' : '15'},
# 'mirror': {'device' : '/dev/hidraw0',
# 'nbmaxtry' : '10',
# 'interval' : '15'},
# 'xbmc_not': {'address' : '192.168.0.20:8080',
# 'delay' : '15',
# 'maxdelay' : '20'},
# 'gagenda': {'email' : "*****@*****.**",
# 'password' : 'XXXXXXXX',
# 'calendarname' : '*****@*****.**',
# 'startup-plugin':'True'},
# 'teleinfo' : {'device' : '/dev/teleinfo',
# 'interval' : '30'},
# 'dawndusk' : {'startup-plugin':'True'},
# 'plcbus' : {'device':'/dev/ttyUSB0'},
# }
self.log.debug("FTC 1")
try:
if key:
self.log.debug("FTC 2")
try:
self.log.debug("Get plg conf for %s / %s / %s" % (techno, hostname, key))
result = self._db.get_plugin_config(techno, hostname, key)
# tricky loop as workaround for a (sqlalchemy?) bug :
# sometimes the given result is for another plugin/key
# so while we don't get the good data, we loop
# This bug happens rarely
while result.id != techno or \
result.hostname != hostname or \
result.key != key:
self.log.debug("Bad result : %s/%s != %s/%s" % (result.id, result.key, plugin, key))
result = self._db.get_plugin_config(techno, hostname, key)
self.log.debug("Get plg conf for %s / %s / %s Result=%s" % (techno, hostname, key, result))
val = result.value
self.log.debug("Get plg conf for %s / %s / %s = %s" % (techno, hostname, key, val))
if val == '':
val = "None"
self.log.debug("Get plg conf for %s / %s / %s = %s (2)" % (techno, hostname, key, val))
return val
except AttributeError:
self.log.debug("Attribute error for %s / %s / %s" % (techno, hostname, key))
return "None"
else:
self.log.debug("FTC 3")
vals = self._db.list_plugin_config(techno, hostname)
res = {}
for val in vals:
if val == '':
res[val.key] = "None"
else:
res[val.key] = val.value
return res
except:
msg = "No config found h=%s, t=%s, k=%s" % (hostname, techno, key)
print(msg)
self.log.warn(msg)
return "None"
def _set_config(self, plugin, hostname, key, value):
'''
Send a config value message for an element's config item
@param plugin : the plugin of the element
@param hostname : hostname
@param key : the key to set
@param value : the value to set
'''
try:
self._db.set_plugin_config(technology, hostname, key, value)
mess = XplMessage()
mess.set_type('xpl-stat')
mess.set_schema('domogik.config')
mess.add_data({'plugin' : plugin})
mess.add_data({'hostname' : hostname})
mess.add_data({'key' : key})
mess.add_data({'value' : value})
self.myxpl.send(mess)
except:
traceback.print_exc()
msg = "Error while setting h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, value)
print(msg)
self.log.warn(msg)
return "None"
class DBConnector(XplPlugin, MQRep):
'''
Manage the connection between database and the plugins
Should be the *only* object along with the StatsManager to access to the database on the core side
'''
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
# Already done in XplPlugin
#MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug(u"Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
with self._db.session_scope():
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}".format(nb_test, DATABASE_CONNECTION_NUM_TRY)
self.log.error(msg)
msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc()))
self.force_leave()
return
self.ready()
# Already done in ready()
#IOLoop.instance().start()
def on_mdp_request(self, msg):
""" Handle Requests over MQ
@param msg : MQ req message
"""
with self._db.session_scope():
# XplPlugin handles MQ Req/rep also
XplPlugin.on_mdp_request(self, msg)
# configuration
if msg.get_action() == "config.get":
self._mdp_reply_config_get(msg)
elif msg.get_action() == "config.set":
self._mdp_reply_config_set(msg)
elif msg.get_action() == "config.delete":
self._mdp_reply_config_delete(msg)
# devices list
elif msg.get_action() == "device.get":
self._mdp_reply_devices_result(msg)
def _mdp_reply_config_get(self, data):
""" Reply to config.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config request not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if 'key' not in msg_data:
get_all_keys = True
key = "*"
else:
get_all_keys = False
key = msg_data['key']
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('key', key) # we let this here to display key or * depending on the case
try:
if get_all_keys == True:
config = self._db.list_plugin_config(name, host)
self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, config))
json_config = {}
for elt in config:
json_config[elt.key] = self.convert(elt.value)
msg.add_data('data', json_config)
else:
value = self._fetch_techno_config(name, host, key)
# temporary fix : should be done in a better way (on db side)
value = self.convert(value)
self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, value))
msg.add_data('value', value)
except:
status = False
reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(type, name, host, key, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_set(self, data):
""" Reply to config.set MQ req
@param data : MQ req message
"""
print "#################"
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config set : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config set not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config set : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config set : missing 'host' field : {0}".format(data)
if 'data' not in msg_data:
status = False
reason = "Config set : missing 'data' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
data = msg_data['data']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
# we add a configured key set to true to tell the UIs and plugins that there are some configuration elements
self._db.set_plugin_config(name, host, "configured", True)
for key in msg_data['data']:
self._db.set_plugin_config(name, host, key, data[key])
self.publish_config_updated(type, name, host)
except:
reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(type, name, host, traceback.format_exc())
status = False
self.log.error(reason)
msg.add_data('status', status)
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_delete(self, data):
""" Reply to config.delete MQ req
Delete all the config items for the given type, name and host
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config request not available for type={0}".format(msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
self._db.del_plugin_config(name, host)
self.log.info(u"Delete config for {0} {1}".format(type, name))
self.publish_config_updated(type, name, host)
except:
status = False
reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(type, name, host, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _fetch_techno_config(self, name, host, key):
'''
Fetch a plugin global config value in the database
@param name : the plugin of the element
@param host : hostname
@param key : the key of the config tuple to fetch
'''
try:
try:
result = self._db.get_plugin_config(name, host, key)
# tricky loop as workaround for a (sqlalchemy?) bug :
# sometimes the given result is for another plugin/key
# so while we don't get the good data, we loop
# This bug happens rarely
while result.id != name or \
result.hostname != host or \
result.key != key:
self.log.debug(u"Bad result : {0}/{1} != {2}/{3}".format(result.id, result.key, plugin, key))
result = self._db.get_plugin_config(name, host, key)
val = result.value
if val == '':
val = "None"
return val
except AttributeError:
# if no result is found
#self.log.error(u"Attribute error : {0}".format(traceback.format_exc()))
return "None"
except:
msg = "No config found host={0}, plugin={1}, key={2}".format(host, name, key)
self.log.warn(msg)
return "None"
def _mdp_reply_devices_result(self, data):
""" Reply to device.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('device.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Devices request : missing 'type' field : {0}".format(data)
if 'name' not in msg_data:
status = False
reason = "Devices request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Devices request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
#if type == "plugin":
# type = DMG_VENDOR_ID
name = msg_data['name']
host = msg_data['host']
dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(type, name, host))
#dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False),
dev_json = dev_list
print(dev_json)
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('devices', dev_json)
self.reply(msg.get())
def convert(self, data):
""" Do some conversions on data
"""
if data == "True":
data = True
if data == "False":
data = False
return data
def publish_config_updated(self, type, name, host):
""" Publish over the MQ a message to inform that a plugin configuration has been updated
@param type : package type (plugin)
@param name : package name
@param host : host
"""
self.log.debug("Publish configuration update notification for {0}-{1}.{2}".format(type, name, host))
self._pub.send_event('plugin.configuration',
{"type" : type,
"name" : name,
"host" : host,
"event" : "updated"})
class ScenarioManager:
""" Manage scenarios : create them, evaluate them, etc ...
A scenario instance contains a condition, which is a boolean
combination of many tests,
and a list of actions
Each test can be :
- test on the test of any device
- test on the time
- action triggered by user (click on UI for ex)
The test on devices are managed directly by xpl Listeners
The test on time will be managed by a TimeManager
The actions will be managed by an ActionManager
{
"condition" :
{ "AND" : {
"OR" : {
"one-uuid" : {
"param_name_1" : {
"token1" : "value",
"token2" : "othervalue"
},
"param_name_2" : {
"token3" : "foo"
}
},
"another-uuid" : {
"param_name_1" : {
"token4" : "bar"
}
}
},
"yet-another-uuid" : {
"param_name_1" : {
"url" : "http://google.fr",
"interval" : "5"
}
}
}
},
"actions" : [
"uid-for-action" : {
"param1" : "value1",
"param2" : "value2"
},
"uid-for-action2" : {
"param3" : "value3"
}
]
}
"""
def __init__(self, log):
""" Create ScenarioManager instance
@param log : Logger instance
"""
# Keep list of conditions as name : instance
self._instances = {}
# an instance of the logger
self.log = log
# load all scenarios from the DB
self._db = DbHelper()
self.load_scenarios()
def load_scenarios(self):
""" Loads all scenarios from the db
for each scenario call the create_scenario method
"""
try:
with self._db.session_scope():
### TEST if database is up
# TODO : move in a function and use it (also used in dbmgr)
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc())
self.log.error(msg)
msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
### Do the stuff
msg = "Connected to the database"
self.log.info(msg)
for scenario in self._db.list_scenario():
self.create_scenario(scenario.name, scenario.json, int(scenario.id), scenario.disabled, scenario.description, scenario.state)
except:
self.log.error(u"Error while loading the scenarios! The error is : {0}".format(traceback.format_exc()))
def shutdown(self):
""" Callback to shut down all parameters
"""
for cond in self._conditions.keys():
self.delete_scenario(cond, db_delete=False)
def get_parsed_condition(self, name):
""" Call cond.get_parsed_condition on the cond with name 'name'
@param name : name of the Condition
@return {'name':name, 'data': parsed_condition} or raise Exception
"""
if name not in self._conditions:
raise KeyError('no key {0} in conditions table'.format(name))
else:
parsed = self._conditions[name].get_parsed_condition()
return {'name': name, 'data': parsed}
def update_scenario(self, cid, name, json_input, dis, desc):
cid = int(cid)
# TODO get the current state and store it
state = True
if cid != 0:
self.del_scenario(cid, False)
return self.create_scenario(name, json_input, cid, dis, desc, state, True)
def del_scenario(self, cid, doDB=True):
try:
cid = int(cid)
if cid == 0 or cid not in self._instances.keys():
self.log.info(u"Scenario deletion : id '{0}' doesn't exist".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)}
else:
self._instances[cid]['instance'].destroy()
del(self._instances[cid])
if doDB:
with self._db.session_scope():
self._db.del_scenario(cid)
self.log.info(u"Scenario {0} deleted".format(cid))
except:
msg = u"Error while deleting the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc())
self.log.error(msg)
return {'status': 'ERROR', 'msg': msg}
def create_scenario(self, name, json_input, cid=0, dis=False, desc=None, state=False, update=False):
""" Create a Scenario from the provided json.
@param name : A name for the condition instance
@param json_input : JSON representation of the condition
The JSON will be parsed to get all the uuids, and test instances will be created.
The json needs to have 2 keys:
- condition => the json that will be used to create the condition instance
- actions => the json that will be used for creating the actions instances
@Return {'name': name} or raise exception
"""
ocid = cid
try:
self.log.info(u"Create or save scenario : name = '{1}', id = '{1}', json = '{2}'".format(name, cid, json_input))
payload = json.loads(json_input) # quick test to check if json is valid
except Exception as e:
self.log.error(u"Creation of a scenario failed, invallid json: {0}".format(json_input))
self.log.error(u"Error is : {0}".format(tracebeck.format_exc()))
return {'status': 'ERROR', 'msg': 'invallid json'}
#if 'IF' not in payload.keys():
# or 'DO' not in payload.keys():
# msg = u"the json for the scenario does not contain condition or actions for scenario {0}".format(name)
# self.log.error(msg)
# return {'status': 'ERROR', 'msg': msg}
# db storage
if int(ocid) == 0:
with self._db.session_scope():
scen = self._db.add_scenario(name, json_input, dis, desc, False)
cid = scen.id
elif update:
with self._db.session_scope():
self._db.update_scenario(cid, name, json_input, dis, desc)
# create the condition itself
try:
scen = ScenarioInstance(self.log, cid, name, payload, dis, state, self._db)
self._instances[cid] = {'name': name, 'json': payload, 'instance': scen, 'disabled': dis }
self.log.debug(u"Create scenario instance {0} with payload {1}".format(name, payload))
self._instances[cid]['instance'].eval_condition()
except Exception as e:
if int(ocid) == 0:
with self._db.session_scope():
self._db.del_scenario(cid)
self.log.error(u"Creation of a scenario failed. Error is : {0}".format(traceback.format_exc()))
return {'status': 'ERROR', 'msg': 'Creation of scenario failed'}
# return
return {'name': name, 'status': 'OK', 'cid': cid}
def eval_condition(self, name):
""" Evaluate a condition calling eval_condition from Condition instance
@param name : The name of the condition instance
@return {'name':name, 'result': evaluation result} or raise Exception
"""
if name not in self._conditions:
raise KeyError('no key {0} in conditions table'.format(name))
else:
res = self._conditions[name].eval_condition()
return {'name': name, 'result': res}
def list_actions(self):
""" Return the list of actions
@return a hash of hashes for the different actions
{ "module1.Action1" : {
"description" : "some description of the action",
"parameters" : { "param1" : {
... see get_expected_entries for details
}
}
"""
self.log.debug("ScenarioManager : list actions")
res = {}
actions = self.__return_list_of_classes(s_a)
for name, cls in actions:
if 'abstract' not in name.lower():
self.log.debug("- {0}".format(name))
inst = cls()
res[name] = {"parameters": inst.get_expected_entries(),
"description": inst.get_description()}
return res
def list_tests(self):
""" Return the list of tests
@return a hash of hashes for the different tests
{ "module1.Test1" : {
"description" : "some description of the test",
"parameters" : { "param1" : {
... see list_parameters doc for detail on this part
}
}
"""
self.log.debug("ScenarioManager : list tests")
res = {}
tests = self.__return_list_of_classes(s_t)
for name, cls in tests:
if 'abstract' not in name.lower():
self.log.debug("- {0}".format(name))
inst = cls(log = self.log)
params = []
for p, i in inst.get_parameters().items():
for param, info in i['expected'].items():
params.append({
"name": "{0}.{1}".format(p, param),
"description": info['description'],
"type": info['type'],
"values": info['values'],
"filters": info['filters'],
})
res[name] = {"parameters": params,
"blockly": inst.get_blockly(),
"description": inst.get_description()}
return res
def list_conditions(self):
""" Return the list of conditions as JSON
"""
ret = []
for cid, inst in self._instances.items():
ret.append({'cid': cid, 'name': inst['name'], 'json': inst['json'], 'disabled': inst['disabled']})
return ret
def enable_scenario(self, cid):
try:
if cid == '' or int(cid) not in self._instances.keys():
self.log.info(u"Scenario enable : id '{0}' doesn't exist".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)}
else:
if self._instances[int(cid)]['instance'].enable():
self._instances[int(cid)]['disabled'] = False
with self._db.session_scope():
self._db.update_scenario(cid, disabled=False)
self.log.info(u"Scenario {0} enabled".format(cid))
return {'status': 'OK', 'msg': u"Scenario {0} enabled".format(cid)}
else:
self.log.info(u"Scenario {0} already enabled".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} already enabled".format(cid)}
except:
msg = u"Error while enabling the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc())
self.log.error(msg)
return {'status': 'ERROR', 'msg': msg}
def disable_scenario(self, cid):
try:
if cid == '' or int(cid) not in self._instances.keys():
self.log.info(u"Scenario disable : id '{0}' doesn't exist".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)}
else:
if self._instances[int(cid)]['instance'].disable():
self._instances[int(cid)]['disabled'] = True
with self._db.session_scope():
self._db.update_scenario(cid, disabled=True)
self.log.info(u"Scenario {0} disabled".format(cid))
return {'status': 'OK', 'msg': u"Scenario {0} disabled".format(cid)}
else:
self.log.info(u"Scenario {0} already disabled".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} already disabled".format(cid)}
except:
msg = u"Error while disabling the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc())
self.log.error(msg)
return {'status': 'ERROR', 'msg': msg}
def test_scenario(self, cid):
try:
if cid == '' or int(cid) not in self._instances.keys():
self.log.info(u"Scenario test : id '{0}' doesn't exist".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)}
else:
self._instances[int(cid)]['instance'].test_actions()
self.log.info(u"Scenario {0} actions called".format(cid))
return {'status': 'OK', 'msg': u"Scenario {0} actions called".format(cid)}
except:
msg = u"Error while calling actions for scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc())
self.log.error(msg)
return {'status': 'ERROR', 'msg': msg}
def __return_list_of_classes(self, package):
""" Return the list of module/classes in a package
@param package : a reference to the package that need to be explored
@return a list of tuple ('modulename.Classname', <instance of class>)
"""
self.log.debug("Get list of classes for package : {0}".format(package))
res = []
mods = pkgutil.iter_modules(package.__path__)
for module in mods:
self.log.debug("- {0}".format(module))
imported_mod = importlib.import_module('.' + module[1], package.__name__)
#get the list of classes in the module
classes = [m for m in inspect.getmembers(imported_mod) if inspect.isclass(m[1])]
# Filter in order to keep only the classes that belong to domogik package and are not abstract
res.extend([(module[1] + "." + c[0], c[1]) for c in filter(
lambda x: x[1].__module__.startswith("domogik.scenario.") and not x[0].startswith("Abstract"), classes)])
return res
class DBConnector(XplPlugin, MQRep):
'''
Manage the connection between database and the plugins
Should be the *only* object along with the StatsManager to access to the database on the core side
'''
def __init__(self):
'''
Initialize database and xPL connection
'''
XplPlugin.__init__(self, 'dbmgr')
# Already done in XplPlugin
#MQRep.__init__(self, zmq.Context(), 'dbmgr')
self.log.debug(u"Init database_manager instance")
# Check for database connexion
self._db = DbHelper()
with self._db.session_scope():
nb_test = 0
db_ok = False
while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY:
nb_test += 1
try:
self._db.list_user_accounts()
db_ok = True
except:
msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}".format(
nb_test, DATABASE_CONNECTION_NUM_TRY)
self.log.error(msg)
msg = "Waiting for {0} seconds".format(
DATABASE_CONNECTION_WAIT)
self.log.info(msg)
time.sleep(DATABASE_CONNECTION_WAIT)
if nb_test >= DATABASE_CONNECTION_NUM_TRY:
msg = "Exiting dbmgr!"
self.log.error(msg)
self.force_leave()
return
msg = "Connected to the database"
self.log.info(msg)
try:
self._engine = self._db.get_engine()
except:
self.log.error(
u"Error while starting database engine : {0}".format(
traceback.format_exc()))
self.force_leave()
return
self.ready()
# Already done in ready()
#IOLoop.instance().start()
def on_mdp_request(self, msg):
""" Handle Requests over MQ
@param msg : MQ req message
"""
with self._db.session_scope():
# XplPlugin handles MQ Req/rep also
XplPlugin.on_mdp_request(self, msg)
# configuration
if msg.get_action() == "config.get":
self._mdp_reply_config_get(msg)
elif msg.get_action() == "config.set":
self._mdp_reply_config_set(msg)
elif msg.get_action() == "config.delete":
self._mdp_reply_config_delete(msg)
# devices list
elif msg.get_action() == "device.get":
self._mdp_reply_devices_result(msg)
def _mdp_reply_config_get(self, data):
""" Reply to config.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config request not available for type={0}".format(
msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if 'key' not in msg_data:
get_all_keys = True
key = "*"
else:
get_all_keys = False
key = msg_data['key']
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data(
'key', key
) # we let this here to display key or * depending on the case
try:
if get_all_keys == True:
config = self._db.list_plugin_config(name, host)
self.log.info(
u"Get config for {0} {1} with key '{2}' : value = {3}".
format(type, name, key, config))
json_config = {}
for elt in config:
json_config[elt.key] = self.convert(elt.value)
msg.add_data('data', json_config)
else:
value = self._fetch_techno_config(name, host, key)
# temporary fix : should be done in a better way (on db side)
value = self.convert(value)
self.log.info(
u"Get config for {0} {1} with key '{2}' : value = {3}".
format(type, name, key, value))
msg.add_data('value', value)
except:
status = False
reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(
type, name, host, key, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_set(self, data):
""" Reply to config.set MQ req
@param data : MQ req message
"""
print "#################"
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config set : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config set not available for type={0}".format(
msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config set : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config set : missing 'host' field : {0}".format(data)
if 'data' not in msg_data:
status = False
reason = "Config set : missing 'data' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
data = msg_data['data']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
# we add a configured key set to true to tell the UIs and plugins that there are some configuration elements
self._db.set_plugin_config(name, host, "configured", True)
for key in msg_data['data']:
self._db.set_plugin_config(name, host, key, data[key])
self.publish_config_updated(type, name, host)
except:
reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(
type, name, host, traceback.format_exc())
status = False
self.log.error(reason)
msg.add_data('status', status)
msg.add_data('reason', reason)
self.log.debug(msg.get())
self.reply(msg.get())
def _mdp_reply_config_delete(self, data):
""" Reply to config.delete MQ req
Delete all the config items for the given type, name and host
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('config.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Config request : missing 'type' field : {0}".format(data)
if msg_data['type'] != "plugin":
status = False
reason = "Config request not available for type={0}".format(
msg_data['type'])
if 'name' not in msg_data:
status = False
reason = "Config request : missing 'name' field : {0}".format(data)
if 'host' not in msg_data:
status = False
reason = "Config request : missing 'host' field : {0}".format(data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
name = msg_data['name']
host = msg_data['host']
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
try:
self._db.del_plugin_config(name, host)
self.log.info(u"Delete config for {0} {1}".format(type, name))
self.publish_config_updated(type, name, host)
except:
status = False
reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(
type, name, host, traceback.format_exc())
self.log.error(reason)
msg.add_data('reason', reason)
msg.add_data('status', status)
self.log.debug(msg.get())
self.reply(msg.get())
def _fetch_techno_config(self, name, host, key):
'''
Fetch a plugin global config value in the database
@param name : the plugin of the element
@param host : hostname
@param key : the key of the config tuple to fetch
'''
try:
try:
result = self._db.get_plugin_config(name, host, key)
# tricky loop as workaround for a (sqlalchemy?) bug :
# sometimes the given result is for another plugin/key
# so while we don't get the good data, we loop
# This bug happens rarely
while result.id != name or \
result.hostname != host or \
result.key != key:
self.log.debug(u"Bad result : {0}/{1} != {2}/{3}".format(
result.id, result.key, plugin, key))
result = self._db.get_plugin_config(name, host, key)
val = result.value
if val == '':
val = "None"
return val
except AttributeError:
# if no result is found
#self.log.error(u"Attribute error : {0}".format(traceback.format_exc()))
return "None"
except:
msg = "No config found host={0}, plugin={1}, key={2}".format(
host, name, key)
self.log.warn(msg)
return "None"
def _mdp_reply_devices_result(self, data):
""" Reply to device.get MQ req
@param data : MQ req message
"""
msg = MQMessage()
msg.set_action('device.result')
status = True
msg_data = data.get_data()
if 'type' not in msg_data:
status = False
reason = "Devices request : missing 'type' field : {0}".format(
data)
if 'name' not in msg_data:
status = False
reason = "Devices request : missing 'name' field : {0}".format(
data)
if 'host' not in msg_data:
status = False
reason = "Devices request : missing 'host' field : {0}".format(
data)
if status == False:
self.log.error(reason)
else:
reason = ""
type = msg_data['type']
#if type == "plugin":
# type = DMG_VENDOR_ID
name = msg_data['name']
host = msg_data['host']
dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(
type, name, host))
#dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False),
dev_json = dev_list
print(dev_json)
msg.add_data('status', status)
msg.add_data('reason', reason)
msg.add_data('type', type)
msg.add_data('name', name)
msg.add_data('host', host)
msg.add_data('devices', dev_json)
self.reply(msg.get())
def convert(self, data):
""" Do some conversions on data
"""
if data == "True":
data = True
if data == "False":
data = False
return data
def publish_config_updated(self, type, name, host):
""" Publish over the MQ a message to inform that a plugin configuration has been updated
@param type : package type (plugin)
@param name : package name
@param host : host
"""
self.log.debug(
"Publish configuration update notification for {0}-{1}.{2}".format(
type, name, host))
self._pub.send_event('plugin.configuration', {
"type": type,
"name": name,
"host": host,
"event": "updated"
})
class ScenarioManager:
""" Manage scenarios : create them, evaluate them, etc ...
A scenario instance contains a condition, which is a boolean
combination of many tests,
and a list of actions
Each test can be :
- test on the test of any device
- test on the time
- action triggered by user (click on UI for ex)
The test on devices are managed directly by xpl Listeners
The test on time will be managed by a TimeManager
The actions will be managed by an ActionManager
{
"condition" :
{ "AND" : {
"OR" : {
"one-uuid" : {
"param_name_1" : {
"token1" : "value",
"token2" : "othervalue"
},
"param_name_2" : {
"token3" : "foo"
}
},
"another-uuid" : {
"param_name_1" : {
"token4" : "bar"
}
}
},
"yet-another-uuid" : {
"param_name_1" : {
"url" : "http://google.fr",
"interval" : "5"
}
}
}
},
"actions" : [
"uid-for-action" : {
"param1" : "value1",
"param2" : "value2"
},
"uid-for-action2" : {
"param3" : "value3"
}
]
}
"""
def __init__(self, log):
""" Create ScenarioManager instance
@param log : Logger instance
"""
# Keep list of conditions as name : instance
self._instances = {}
# an instance of the logger
self.log = log
# load all scenarios from the DB
self._db = DbHelper()
self.load_scenarios()
def load_scenarios(self):
""" Loads all scenarios from the db
for each scenario call the create_scenario method
"""
with self._db.session_scope():
for scenario in self._db.list_scenario():
self.create_scenario(scenario.name, scenario.json, int(scenario.id), scenario.disabled, scenario.description)
def shutdown(self):
""" Callback to shut down all parameters
"""
for cond in self._conditions.keys():
self.delete_scenario(cond, db_delete=False)
def get_parsed_condition(self, name):
""" Call cond.get_parsed_condition on the cond with name 'name'
@param name : name of the Condition
@return {'name':name, 'data': parsed_condition} or raise Exception
"""
if name not in self._conditions:
raise KeyError('no key %s in conditions table' % name)
else:
parsed = self._conditions[name].get_parsed_condition()
return {'name': name, 'data': parsed}
def update_scenario(self, cid, name, json_input, dis, desc):
if int(cid) != 0:
self.del_scenario(cid, False)
return self.create_scenario(name, json_input, cid, dis, desc, True)
def del_scenario(self, cid, doDB=True):
try:
if cid == '' or int(cid) not in self._instances.keys():
self.log.info(u"Scenario deletion : id '{0}' doesn't exist".format(cid))
return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)}
else:
self._instances[int(cid)]['instance'].destroy()
del(self._instances[int(cid)])
if doDB:
with self._db.session_scope():
self._db.del_scenario(cid)
self.log.info(u"Scenario {0} deleted".format(cid))
except:
msg = u"Error while deleting the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc())
self.log.error(msg)
return {'status': 'ERROR', 'msg': msg}
def create_scenario(self, name, json_input, cid=0, dis=False, desc=None, update=False):
""" Create a Scenario from the provided json.
@param name : A name for the condition instance
@param json_input : JSON representation of the condition
The JSON will be parsed to get all the uuids, and test instances will be created.
The json needs to have 2 keys:
- condition => the json that will be used to create the condition instance
- actions => the json that will be used for creating the actions instances
@Return {'name': name} or raise exception
"""
ocid = cid
try:
self.log.info(u"Create or save scenario : name = '{1}', id = '{1}', json = '{2}'".format(name, cid, json_input))
payload = json.loads(json_input) # quick test to check if json is valid
except Exception as e:
self.log.error(u"Creation of a scenario failed, invallid json: {0}".format(json_input))
self.log.debug(e)
return {'status': 'NOK', 'msg': 'invallid json'}
if 'IF' not in payload.keys() \
or 'DO' not in payload.keys():
msg = u"the json for the scenario does not contain condition or actions for scenario {0}".format(name)
self.log.error(msg)
return {'status': 'NOK', 'msg': msg}
# db storage
if int(ocid) == 0:
with self._db.session_scope():
scen = self._db.add_scenario(name, json_input, dis, desc)
cid = scen.id
elif update:
with self._db.session_scope():
self._db.update_scenario(cid, name, json_input, dis, desc)
# create the condition itself
try:
scen = ScenarioInstance(self.log, cid, name, payload, dis)
self._instances[cid] = {'name': name, 'json': payload, 'instance': scen }
self.log.debug(u"Create scenario instance {0} with payload {1}".format(name, payload['IF']))
self._instances[cid]['instance'].eval_condition()
except Exception as e:
if int(ocid) == 0:
with self._db.session_scope():
self._db.del_scenario(cid)
self.log.error(u"Creation of a scenario failed")
self.log.debug(e)
return {'status': 'NOK', 'msg': 'Creation of scenario failed'}
# return
return {'name': name, 'cid': cid}
def eval_condition(self, name):
""" Evaluate a condition calling eval_condition from Condition instance
@param name : The name of the condition instance
@return {'name':name, 'result': evaluation result} or raise Exception
"""
if name not in self._conditions:
raise KeyError('no key %s in conditions table' % name)
else:
res = self._conditions[name].eval_condition()
return {'name': name, 'result': res}
def trigger_actions(self, name):
""" Trigger that will be called when a condition evaluates to True
"""
if name not in self._conditions_actions \
or name not in self._conditions:
raise KeyError('no key %s in one of the _conditions tables table' % name)
else:
for action in self._conditions_actions[name]:
self._actions_mapping[action].do_action( \
self._conditions[name], \
self._conditions[name].get_mapping() \
)
def list_actions(self):
""" Return the list of actions
@return a hash of hashes for the different actions
{ "module1.Action1" : {
"description" : "some description of the action",
"parameters" : { "param1" : {
... see get_expected_entries for details
}
}
"""
self.log.debug("ScenarioManager : list actions")
res = {}
actions = self.__return_list_of_classes(s_a)
for name, cls in actions:
self.log.debug("- {0}".format(name))
inst = cls()
res[name] = {"parameters": inst.get_expected_entries(),
"description": inst.get_description()}
return res
def list_tests(self):
""" Return the list of tests
@return a hash of hashes for the different tests
{ "module1.Test1" : {
"description" : "some description of the test",
"parameters" : { "param1" : {
... see list_parameters doc for detail on this part
}
}
"""
self.log.debug("ScenarioManager : list tests")
res = {}
tests = self.__return_list_of_classes(s_t)
for name, cls in tests:
self.log.debug("- {0}".format(name))
inst = cls(log = self.log)
params = []
for p, i in inst.get_parameters().iteritems():
for param, info in i['expected'].iteritems():
params.append({
"name": "{0}.{1}".format(p, param),
"description": info['description'],
"type": info['type'],
"values": info['values'],
"filters": info['filters'],
})
res[name] = {"parameters": params,
"description": inst.get_description()}
return res
#for name, cls in tests:
# self.log.debug("- {0}".format(name))
# inst = cls(log = self.log)
# res[name] = []
# for p, i in inst.get_parameters().iteritems():
# for param, info in i['expected'].iteritems():
# res[name].append({
# "name": "{0}.{1}".format(p, param),
# "description": info['description'],
# "type": info['type'],
# "values": info['values'],
# "filters": info['filters'],
# })
# inst.destroy()
#return res
def list_conditions(self):
""" Return the list of conditions as JSON
"""
ret = []
for cid, inst in self._instances.iteritems():
ret.append({'cid': cid, 'name': inst['name'], 'json': inst['json']})
return ret
def __return_list_of_classes(self, package):
""" Return the list of module/classes in a package
@param package : a reference to the package that need to be explored
@return a list of tuple ('modulename.Classname', <instance of class>)
"""
self.log.debug("Get list of classes for package : {0}".format(package))
res = []
mods = pkgutil.iter_modules(package.__path__)
for module in mods:
self.log.debug("- {0}".format(module))
imported_mod = importlib.import_module('.' + module[1], package.__name__)
#get the list of classes in the module
classes = [m for m in inspect.getmembers(imported_mod) if inspect.isclass(m[1])]
# Filter in order to keep only the classes that belong to domogik package and are not abstract
res.extend([(module[1] + "." + c[0], c[1]) for c in filter(
lambda x: x[1].__module__.startswith("domogik.scenario.") and not x[0].startswith("Abstract"), classes)])
return res
ret = {}
for d in devs:
d = list(d)
if d[0] not in ret:
ret[d[0]] = {}
ret[d[0]]['name'] = d[1]
ret[d[0]]['keys'] = []
ret[d[0]]['keys'].append(d[2])
else:
ret[d[0]]['keys'].append(d[2])
return ret
if __name__ == "__main__":
# 0- connect to the DB
db = DbHelper()
db.open_session()
# 1- list current devices (odl ones)
do = True
old_devs = corellateOld(db.upgrade_list_old())
while do:
print old_devs
# show dev menu
i = 1
for dev in old_devs:
print("{0}. {1}".format(i, old_devs[dev]['name']))
i = i + 1
print("0. Exit")
sel = i + 1
while sel > i:
