def upgrade(): db = DbHelper() with db.session_scope(): db.add_user_account_with_person('Anonymous', 'Anonymous', 'Rest', 'Anonymous') del db pass
def __init__(self, json_path): """ Init tool @param plugin_name : plugin name """ self._db = DbHelper() try: self.pkg = PackageJson(path = json_path).json except PackageException as exp: print("Error in json file:") print( exp.value ) exit() except: print(str(traceback.format_exc())) return print("Json file OK") # check type == plugin if self.pkg["identity"]["type"] not in ["plugin", "external"]: print("Error : this package type is not recognized") exit() # check if json version is at least 2 if self.pkg['json_version'] < 2: print("Error : this package is to old for this version of domogik") exit()
def __init__(self, log, queue, conv, pub): threading.Thread.__init__(self) self._db = DbHelper() self._log = log self._queue = queue self._conv = conv self._pub = pub
def test_user_config_file(user_home, user_entry): info("Check user config file contents") import ConfigParser config = ConfigParser.ConfigParser() config.read("/etc/domogik/domogik.cfg") #check [domogik] section dmg = dict(config.items('domogik')) database = dict(config.items('database')) admin = dict(config.items('admin')) butler = dict(config.items('butler')) backup = dict(config.items('backup')) ok("Config file correctly loaded") info("Parse [domogik] section") import domogik #Check ix xpl port is not used _check_port_availability("0.0.0.0", 3865, udp = True) ok("xPL hub IP/port is not bound by anything else") parent_conn, child_conn = Pipe() p = Process(target=_test_user_can_write, args=(child_conn, dmg['log_dir_path'],user_entry,)) p.start() p.join() assert parent_conn.recv(), "The directory %s for log does not exist or does not have right permissions" % dmg['log_dir_path'] assert dmg['log_level'] in ['debug','info','warning','error','critical'], "The log_level parameter does not have a good value. Must \ be one of debug,info,warning,error,critical" ### obsolete #if not os.path.isdir(dmg['src_prefix'] + '/share/domogik'): # try: # f = os.listdir("%s/share/domogik" % dmg['src_prefix']) # f.close() # except OSError: # fail("Can't access %s/share/domogik. Check %s is available for domogik user (if you are in development mode, be sure the directory which contains the sources is available for domogik user)." % (dmg['src_prefix'],dmg['src_prefix'])) # exit() ok("[domogik] section seems good") # check [database] section info("Parse [database] section") assert database['type'] == 'mysql', "Only mysql database type is supported at the moment" uid = user_entry.pw_uid os.setreuid(0,uid) old_home = os.environ['HOME'] os.environ['HOME'] = user_home from domogik.common.database import DbHelper d = DbHelper() os.setreuid(0,0) os.environ['HOME'] = old_home assert d.get_engine() != None, "Engine is not set, it seems something went wrong during connection to the database" ok("[database] section seems good") # Check [admin] section info("Parse [admin] section") for ipadd in get_ip_for_interfaces(admin['interfaces'].split(",")): _check_port_availability(ipadd, admin['port']) ok("Admin server IP/port is not bound by anything else")
def __init__(self, log=None, trigger=None, cond=None, params=None): AbstractTest.__init__(self, log, trigger, cond, params) self._sensorId = params self.set_description("Check The value for a sensor with id {0}".format( self._sensorId)) self.log = log self._db = DbHelper() self._res = None self._dataType = None self._dt_parent = None # get info from db with self._db.session_scope(): sensor = self._db.get_sensor(self._sensorId) if sensor is not None: self._res = sensor.last_value self._dataType = sensor.data_type # find the parent dt type if cond: dt_parent = self._dataType while 'parent' in cond.datatypes[ dt_parent] and cond.datatypes[dt_parent]['parent'] != None: dt_parent = cond.datatypes[dt_parent]['parent'] self._dt_parent = dt_parent # set new val self._res = self._convert(self._res) # start the thread self._event = Event() self._fetch_thread = Thread(target=self._fetch, name="pollthread") self._fetch_thread.start()
def _callback(self, message): """ Callback for the xpl message @param message : the Xpl message received """ self._log_stats.debug("Stat received for device {0}." \ .format(self._dev['name'])) current_date = calendar.timegm(time.gmtime()) stored_value = None try: # find what parameter to store for param in self._stat.params: # self._log_stats.debug("Checking param {0}".format(param)) if param.sensor_id is not None and param.static is False: if param.key in message.data: value = message.data[param.key] # self._log_stats.debug( \ # "Key found {0} with value {1}." \ # .format(param.key, value)) store = True if param.ignore_values: if value in eval(param.ignore_values): self._log_stats.debug( \ "Value {0} is in the ignore list {0}, so not storing." \ .format(value, param.ignore_values)) store = False if store: # check if we need a conversion if self._sen.conversion is not None and self._sen.conversion != '': value = call_package_conversion(\ self._log_stats, \ self._dev['client_id'], \ self._sen.conversion, \ value) self._log_stats.info( \ "Storing stat for device '{0}' ({1}) and sensor'{2}' ({3}): key '{4}' with value '{5}' after conversion." \ .format(self._dev['name'], self._dev['id'], self._sen.name, self._sen.id, param.key, value)) # do the store stored_value = value my_db = DbHelper() with my_db.session_scope(): my_db.add_sensor_history(\ param.sensor_id, \ value, \ current_date) del(my_db) else: self._log_stats.debug("Don't need to store this value") #else: # self._log_stats.debug("Key not found in message data") #else: # self._log_stats.debug("No sensor attached") except: self._log_stats.error(traceback.format_exc()) # publish the result self._pub.send_event('device-stats', \ {"timestamp" : current_date, \ "device_id" : self._dev['id'], \ "sensor_id" : self._sen.id, \ "stored_value" : stored_value})
def __init__(self, log, queue, storeQueue): threading.Thread.__init__(self) self._db = DbHelper() self._log = log self._queue = queue self._queue_store = storeQueue # on startup, load the device parameters self.on_device_changed()
def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw', log_prefix="core_") self.add_mq_sub('client.conversion') self.add_mq_sub('client.list') self.add_mq_sub('client.sensor') self.add_mq_sub('device.update') self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') # some initial data sets self.client_xpl_map = {} self.client_conversion_map = {} self._db_sensors = {} self._db_xplstats = {} # load devices informations self._reload_devices() self._reload_commands() self._reload_xpl_stats() # queue to store the message that needs to be ahndled for sensor checking self._sensor_queue = queue.Queue() # queue to handle the sensor storage self._sensor_store_queue = queue.Queue() # all command handling params # _lock => to be sure to be thread safe # _dict => uuid to xplstat translationg # _pkt => received messages to check self._cmd_lock_d = threading.Lock() self._cmd_dict = {} self._cmd_lock_p = threading.Lock() self._cmd_pkt = {} # load some initial data from manager and db self._load_client_to_xpl_target() self._load_conversions() # create a general listener self._create_xpl_trigger() # start handling the xplmessages self._x_thread = self._XplSensorThread(\ self.log, self._sensor_queue, \ self._sensor_store_queue) self._x_thread.start() # start handling the command reponses in a thread self._c_thread = self._XplCommandThread(\ self.log, self._db, self._cmd_lock_d, \ self._cmd_lock_p, self._cmd_dict, self._cmd_pkt, self.pub) self._c_thread.start() # start the sensor storage thread self._s_thread = self._SensorStoreThread(\ self._sensor_store_queue, self.log, \ self._get_conversion_map, self.pub) self._s_thread.start() # start the sensorthread self.ready()
def __init__(self, queue, log, get_conversion_map, pub): threading.Thread.__init__(self) self._log = log self._db = DbHelper() self._conv = get_conversion_map self._queue = queue self._pub = pub # on startup, load the device parameters self.on_device_changed()
def __init__(self, log): """ Create ScenarioManager instance @param log : Logger instance """ # Keep list of conditions as name : instance self._instances = {} # an instance of the logger self.log = log # load all scenarios from the DB self._db = DbHelper() self.load_scenarios()
def __init__(self): ''' Initialize database and xPL connection ''' XplPlugin.__init__(self, 'dbmgr') MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug("Init database_manager instance") # Check for database connexion self._db = DbHelper() nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % ( nb_test, DATABASE_CONNECTION_NUM_TRY) print(msg) self.log.error(msg) msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT print(msg) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" print(msg) self.log.error(msg) self.force_leave() return msg = "Connected to the database" print(msg) self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error("Error while starting database engine : %s" % traceback.format_exc()) self.force_leave() return Listener(self._request_config_cb, self.myxpl, { 'schema': 'domogik.config', 'xpltype': 'xpl-cmnd' }) self.enable_hbeat() IOLoop.instance().start()
def start_http(self): """ Start HTTP Server """ self.log.info(u"Start HTTP Server on %s:%s..." % (self.interfaces, self.port)) # logger for log in self.log.handlers: admin_app.logger.addHandler(log) admin_app.zmq_context = self.zmq admin_app.db = DbHelper() admin_app.datatypes = self.datatypes tapp = Application([ (r"/ws", AdminWebSocket), (r".*", FallbackHandler, dict(fallback=WSGIContainer(admin_app))) ]) # create the server # for ssl, extra parameter to HTTPServier init if self.use_ssl is True: ssl_options = { "certfile": self.cert_file, "keyfile": self.key_file, } self.http_server = HTTPServer(tapp, ssl_options=ssl_options) else: self.http_server = HTTPServer(tapp) # listen on the interfaces if self.interfaces != "": intf = self.interfaces.split(',') for ip in get_ip_for_interfaces(intf): self.http_server.listen(int(self.port), address=ip) else: self.http_server.bind(int(self.port)) self.http_server.start(1) return
def start_http(self): """ Start HTTP Server """ self.log.info(u"Start HTTP Server on {0}:{1}...".format( self.interfaces, self.port)) # logger for log in self.log.handlers: admin_app.logger.addHandler(log) admin_app.zmq_context = self.zmq admin_app.db = DbHelper() admin_app.datatypes = self.datatypes admin_app.clean_json = self.clean_json admin_app.rest_auth = self.rest_auth admin_app.apiversion = REST_API_VERSION admin_app.use_ssl = self.use_ssl admin_app.interfaces = self.interfaces admin_app.port = self.port admin_app.hostname = self.get_sanitized_hostname() admin_app.resources_directory = self.get_resources_directory() admin_app.packages_directory = self.get_packages_directory() admin_app.publish_directory = self.get_publish_directory( ) # publish directory for all packages publisher = Publisher() tapp = Application([(r"/ws", Subscription, dict(publisher=publisher)), (r".*", FallbackHandler, dict(fallback=WSGIContainer(admin_app)))]) # create the server # for ssl, extra parameter to HTTPServier init if self.use_ssl is True: ssl_options = { "certfile": self.cert_file, "keyfile": self.key_file, } print(ssl_options) self.http_server = HTTPServer(tapp, ssl_options=ssl_options) else: self.http_server = HTTPServer(tapp) # listen on the interfaces if self.interfaces != "": # value can be : lo, eth0, ... # or also : '*' to catch all interfaces, whatever they are intf = self.interfaces.split(',') self.log.info( "The admin will be available on the below addresses : ") num_int = 0 for ip in get_ip_for_interfaces(intf, log=self.log): self.log.info(" - {0}:{1} [BIND]".format(ip, self.port)) self.http_server.listen(int(self.port), address=ip) num_int += 1 if num_int == 0: self.log.error( "The admin is not configured to use any working network interface! Please check configuration!!!!!!" ) else: self.http_server.bind(int(self.port)) self.http_server.start(0) yield [publisher.publishToMQ(), publisher.publishToWS()]
def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw', log_prefix = "") MQAsyncSub.__init__(self, self.zmq, 'xplgw', ['client.conversion', 'client.list']) self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') self.stats = None self.client_xpl_map = {} self.client_conversion_map = {} self._load_client_to_xpl_target() self._load_conversions() self.load() self.ready()
def __init__(self, handler_params, xpl): """ @param handler_params : The server params @param xpl : A xPL Manager instance """ try: self.myxpl = xpl # logging initialization log = logger.Logger('rest-stat') self._log_stats = log.get_logger('rest-stat') self._log_stats.info("Rest Stat Manager initialisation...") # create the dbHelper self._db = DbHelper() ### Rest data self.handler_params = handler_params self.handler_params.append(self._log_stats) self.handler_params.append(self._db) self._event_requests = self.handler_params[0]._event_requests self.get_exception = self.handler_params[0].get_exception self.stats = None except : self._log_stats.error("%s" % traceback.format_exc())
def __init__(self): ''' Initialize database and xPL connection ''' Plugin.__init__(self, 'dbmgr', log_prefix='core_') # Already done in Plugin #MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug(u"Init database_manager instance") # Check for database connexion self._db = DbHelper() with self._db.session_scope(): # TODO : move in a function and use it (also used in dbmgr) nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format( nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc()) self.log.error(msg) msg = "Waiting for {0} seconds".format( DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return msg = "Connected to the database" self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error( u"Error while starting database engine : {0}".format( traceback.format_exc())) self.force_leave() return self.ready()
def _request_config_cb(self, message): ''' Callback to receive a request for some config stuff @param message : the xPL message ''' #try: self._db = DbHelper(engine=self._engine) techno = message.data['plugin'] hostname = message.data['hostname'] key = message.data['key'] msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key) print(msg) self.log.debug(msg) if "value" in message.data: new_value = message.data['value'] else: new_value = None if "element" in message.data: element = message.data['element'] else: element = None msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key) print(msg) self.log.debug(msg) # Set configuration if new_value: msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, new_value) print msg self.log.debug(msg) self._set_config(techno, hostname, key, new_value) # Send configuration else: msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key) print(msg) self.log.debug(msg) if element: msg = "Request h=%s, t=%s, k=%s (send if element)" % (hostname, techno, key) print(msg) self.log.debug(msg) self._send_config(techno, hostname, key, self._fetch_elmt_config(techno, element, key), element) else: msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname, techno, key) print(msg) self.log.debug(msg) if not key: msg = "Request h=%s, t=%s, k=%s (send if not key)" % (hostname, techno, key) print(msg) self.log.debug(msg) keys = self._fetch_techno_config(techno, hostname, key).keys() values = self._fetch_techno_config(techno, hostname, key).values() self._send_config(techno, hostname, keys, values) else: msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % (hostname, techno, key) print(msg) self.log.debug(msg) self._send_config(techno, hostname, key, self._fetch_techno_config(techno, hostname, key))
def __init__(self, log, db, lock_d, lock_p, dic, pkt, pub): threading.Thread.__init__(self) self._db = DbHelper() self._log = log self._lock_d = lock_d self._lock_p = lock_p self._dict = dic self._pkt = pkt self._pub = pub
def _callback(self, message): """ Callback for the xpl message @param message : the Xpl message received """ my_db = DbHelper() self._log_stats.debug("Stat received for device %s." \ % (self._dev.name)) current_date = calendar.timegm(time.gmtime()) device_data = [] try: # find what parameter to store for p in self._stat.params: if p.sensor_id is not None: if p.key in message.data: value = message.data[p.key] self._log_stats.debug("Key found %s with value %s." \ % (p.key, value)) store = True if p.ignore_values: if value in eval(p.ignore_values): self._log_stats.debug("Value %s is in the ignore list %s, so not storing." \ % (value, p.ignore_values)) store = False if store: # check if we need a conversion if self._sen.conversion is not None and self._sen.conversion != '': value = call_package_conversion(\ self._log_stats, self._dev.device_type.plugin_id, \ self._sen.conversion, value) self._log_stats.debug("Key found %s with value %s after conversion." \ % (p.key, value)) # do the store device_data.append({"value" : value, "sensor": p.sensor_id}) my_db.add_sensor_history(p.sensor_id, value, current_date) except: error = "Error when processing stat : %s" % traceback.format_exc() print("==== Error in Stats ====") print(error) print("========================") self._log_stats.error(error) # put data in the event queue self._event_requests.add_in_queues(self._dev.id, {"timestamp" : current_date, "device_id" : self._dev.id, "data" : device_data}) del(my_db)
def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw') self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') self.stats = None self.client_xpl_map = {} self._load_client_to_xpl_target() self.load() self.ready()
def __init__(self, log = None, trigger = None, cond = None, params = None): AbstractTest.__init__(self, log, trigger, cond, params) self._subMessages.append( 'device-stats' ) self._sensorId = params self.set_description(u"Check if value/date changes for a sensor with id {0}".format(self._sensorId)) self.log = log self._db = DbHelper() self._res = None self._dataType = None self._dt_parent = None # get initital info from db with self._db.session_scope(): sensor = self._db.get_sensor(self._sensorId) if sensor is not None: self._res = sensor.last_value self._dataType = sensor.data_type # find the parent dt type if cond: dt_parent = self._dataType while 'parent' in cond.datatypes[dt_parent] and cond.datatypes[dt_parent]['parent'] != None: dt_parent = cond.datatypes[dt_parent]['parent'] self._dt_parent = dt_parent
def start_http(self): """ Start HTTP Server """ self.log.info(u"Start HTTP Server on %s:%s..." % (self.interfaces, self.port)) # logger for log in self.log.handlers: urlHandler.logger.addHandler(log) # db access urlHandler.db = DbHelper() # needed for status urlHandler.apiversion = self._rest_api_version urlHandler.use_ssl = self.use_ssl urlHandler.hostname = self.get_sanitized_hostname() urlHandler.clean_json = self.clean_json urlHandler.zmq_context = self.zmq # handler for getting the paths urlHandler.resources_directory = self.get_resources_directory() # create the server # for ssl, extra parameter to HTTPServier init if self.use_ssl: ssl_options = { "certfile": self.cert_file, "keyfile": self.key_file, } self.http_server = HTTPServer(WSGIContainer(urlHandler), ssl_options=ssl_options) else: self.http_server = HTTPServer(WSGIContainer(urlHandler)) # listen on the interfaces if self.interfaces != "": intf = self.interfaces.split(',') self.log.info( "REST server will be available on the below addresses : ") num_int = 0 for ip in get_ip_for_interfaces(intf, log=self.log): self.log.info(" - {0}:{1} [BIND]".format(ip, self.port)) self.http_server.listen(int(self.port), address=ip) num_int += 1 if num_int == 0: self.log.error( "The rest server is not configured to use any working network interface! Please check configuration!!!!!!" ) else: self.http_server.bind(int(self.port)) self.http_server.start(1) return
class AbstractSensorTest(AbstractTest): def __init__(self, log = None, trigger = None, cond = None, params = None): AbstractTest.__init__(self, log, trigger, cond, params) self._subMessages.append( 'device-stats' ) self._sensorId = params self.set_description(u"Check if value/date changes for a sensor with id {0}".format(self._sensorId)) self.log = log self._db = DbHelper() self._res = None self._dataType = None self._dt_parent = None # get initital info from db with self._db.session_scope(): sensor = self._db.get_sensor(self._sensorId) if sensor is not None: self._res = sensor.last_value self._dataType = sensor.data_type # find the parent dt type if cond: dt_parent = self._dataType while 'parent' in cond.datatypes[dt_parent] and cond.datatypes[dt_parent]['parent'] != None: dt_parent = cond.datatypes[dt_parent]['parent'] self._dt_parent = dt_parent def destroy(self): """ Destroy fetch thread """ AbstractTest.destroy(self) def on_message(self, did, msg): """Receive message from MQ sub""" if self._sensorId: if 'sensor_id' in msg: if int(msg['sensor_id']) == int(self._sensorId): self.log.debug(u"{0} : received MQ message : {1}. Please notice that if a scenario which use several of this sensor block is currently being evaluated, only one value is processed (in case the sensor values change too fast!)".format(self.__class__.__name__, msg)) self.handle_message(did, msg)
def __init__(self): ''' Initialize database and xPL connection ''' XplPlugin.__init__(self, 'dbmgr') MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug("Init database_manager instance") # Check for database connexion self._db = DbHelper() nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (nb_test, DATABASE_CONNECTION_NUM_TRY) print(msg) self.log.error(msg) msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT print(msg) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" print(msg) self.log.error(msg) self.force_leave() return msg = "Connected to the database" print(msg) self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error("Error while starting database engine : %s" % traceback.format_exc()) self.force_leave() return Listener(self._request_config_cb, self.myxpl, {'schema': 'domogik.config', 'xpltype': 'xpl-cmnd'}) self.enable_hbeat() IOLoop.instance().start()
def __init__(self, json_path): """ Init tool @param plugin_name : plugin name """ self._db = DbHelper() try: self.pkg = PackageJson(path = json_path).json except: print(str(traceback.format_exc())) return print("Json file OK") # check type == plugin if self.pkg["identity"]["type"] not in ["plugin", "external"]: print("Error : this package type is not recognized") exit()
def __init__(self): ''' Initialize database and xPL connection ''' Plugin.__init__(self, 'dbmgr', log_prefix='core_') # Already done in Plugin #MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug(u"Init database_manager instance") # Check for database connexion self._db = DbHelper() with self._db.session_scope(): # TODO : move in a function and use it (also used in dbmgr) nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc()) self.log.error(msg) msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return msg = "Connected to the database" self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc())) self.force_leave() return self.ready()
def __init__(self, log): """ Create ScenarioManager instance @param log : Logger instance """ # Keep uuid <-> instance mapping for tests and actions self._tests_mapping = {} self._actions_mapping = {} # Keep list of conditions as name : instance self._conditions = {} # Keep list of actions uuid linked to a condition as name : [uuid1, uuid2, ... ] self._conditions_actions = {} # an instance of the logger self.log = log # As we lazy-load all the tests/actions in __instanciate # we need to keep the module in a list so that we don't need to reload them # every time. self._test_cache = {} self._action_cache = {} # load all scenarios from the DB self._db = DbHelper() self.load_scenarios()
def start_http(self): """ Start HTTP Server """ self.log.info(u"Start HTTP Server on %s:%s..." % (self.interfaces, self.port)) # logger for log in self.log.handlers: urlHandler.logger.addHandler(log) # db access urlHandler.db = DbHelper() # needed for status urlHandler.apiversion = self._rest_api_version urlHandler.use_ssl = self.use_ssl urlHandler.hostname = self.get_sanitized_hostname() urlHandler.clean_json = self.clean_json # reload statsmanager helper urlHandler.reload_stats = self.reload_stats urlHandler.zmq_context = self.zmq # handler for getting the paths urlHandler.resources_directory = self.get_resources_directory() # create the server # for ssl, extra parameter to HTTPServier init if self.use_ssl: ssl_options = { "certfile": self.cert_file, "keyfile": self.key_file, } self.http_server = HTTPServer(WSGIContainer(urlHandler), ssl_options=ssl_options) else: self.http_server = HTTPServer(WSGIContainer(urlHandler)) # listen on the interfaces if self.interfaces != "": intf = self.interfaces.split(',') for ip in get_ip_for_interfaces(intf): self.http_server.listen(int(self.port), address=ip) else: self.http_server.bind(int(self.port)) self.http_server.start(1) return
class _SensorThread(threading.Thread): """ SensorThread class Class that will handle the sensor storage in a seperated thread This will get messages from the SensorQueue """ def __init__(self, log, queue, conv, pub): threading.Thread.__init__(self) self._db = DbHelper() self._log = log self._queue = queue self._conv = conv self._pub = pub def _find_storeparam(self, item): #print("ITEM = {0}".format(item['msg'])) found = False tostore = [] for xplstat in self._db.get_all_xpl_stat(): sensors = 0 matching = 0 statics = 0 if xplstat.schema == item["msg"].schema: #print(" XPLSTAT = {0}".format(xplstat)) # we found a possible xplstat # try to match all params and try to find a sensorid and a vlaue to store for param in xplstat.params: #print(" PARAM = {0}".format(param)) ### Caution ! # in case you, who are reading this, have to debug something like that : # 2015-08-16 22:04:26,190 domogik-xplgw INFO Storing stat for device 'Garage' (6) and sensor 'Humidity' (69): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,306 domogik-xplgw INFO Storing stat for device 'Salon' (10) and sensor 'Humidity' (76): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,420 domogik-xplgw INFO Storing stat for device 'Chambre d'Ewan' (11) and sensor 'Humidity' (80): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,533 domogik-xplgw INFO Storing stat for device 'Chambre des parents' (12) and sensor 'Humidity' (84): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,651 domogik-xplgw INFO Storing stat for device 'Chambre de Laly' (13) and sensor 'Humidity' (88): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,770 domogik-xplgw INFO Storing stat for device 'Entrée' (17) and sensor 'Humidity' (133): key 'current' with value '53' after conversion. # # which means that for a single xPL message, the value is stored in several sensors (WTF!!! ?) # It can be related to the fact that the device address key is no more corresponding between the plugin (info.json and xpl sent by python) and the way the device was create in the databse # this should not happen, but in case... well, we may try to find a fix... if param.key in item["msg"].data and param.static: statics = statics + 1 if param.multiple is not None and len(param.multiple) == 1 and item["msg"].data[param.key] in param.value.split(param.multiple): matching = matching + 1 elif item["msg"].data[param.key] == param.value: matching = matching + 1 # now we have a matching xplstat, go and find all sensors if matching == statics: #print(" MATHING !!!!!") for param in xplstat.params: if param.key in item["msg"].data and not param.static: #print(" ===> TOSTORE !!!!!!!!! : {0}".format({'param': param, 'value': item["msg"].data[param.key]})) tostore.append( {'param': param, 'value': item["msg"].data[param.key]} ) if len(tostore) > 0: found = True if found: return (found, tostore) else: return False def run(self): while True: try: item = self._queue.get() self._log.debug(u"Getting item from the sensorQueue, current length = {0}".format(self._queue.qsize())) # if clientid is none, we don't know this sender so ignore # TODO check temp disabled until external members are working #if item["clientId"] is not None: if True: with self._db.session_scope(): fdata = self._find_storeparam(item) if fdata: #// ICI !!! self._log.debug(u"Found a matching sensor, so starting the storage procedure. Sensor : {0}".format(fdata)) for data in fdata[1]: value = data['value'] storeparam = data['param'] current_date = calendar.timegm(time.gmtime()) store = True if storeparam.ignore_values: if value in eval(storeparam.ignore_values): self._log.debug(u"Value {0} is in the ignore list {0}, so not storing.".format(value, storeparam.ignore_values)) store = False if store: # get the sensor and dev sen = self._db.get_sensor(storeparam.sensor_id) dev = self._db.get_device(sen.device_id) # check if we need a conversion if sen.conversion is not None and sen.conversion != '': if dev['client_id'] in self._conv and sen.conversion in self._conv[dev['client_id']]: self._log.debug( \ u"Calling conversion {0}".format(sen.conversion)) exec(self._conv[dev['client_id']][sen.conversion]) value = locals()[sen.conversion](value) self._log.info( \ u"Storing stat for device '{0}' ({1}) and sensor '{2}' ({3}): key '{4}' with value '{5}' after conversion." \ .format(dev['name'], dev['id'], sen.name, sen.id, storeparam.key, value)) # do the store try: self._db.add_sensor_history(\ storeparam.sensor_id, \ value, \ current_date) except Exception as exp: self._log.error(u"Error when adding sensor history : {0}".format(traceback.format_exc())) else: self._log.debug(u"Don't need to store this value") # publish the result self._pub.send_event('device-stats', \ {"timestamp" : current_date, \ "device_id" : dev['id'], \ "sensor_id" : sen.id, \ "stored_value" : value}) except Queue.Empty: # nothing in the queue, sleep for 1 second time.sleep(1) except Exception as exp: self._log.error(traceback.format_exc())
def _request_config_cb(self, message): ''' Callback to receive a request for some config stuff @param message : the xPL message ''' #try: self._db = DbHelper(engine=self._engine) techno = message.data['plugin'] hostname = message.data['hostname'] key = message.data['key'] msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key) print(msg) self.log.debug(msg) if "value" in message.data: new_value = message.data['value'] else: new_value = None if "element" in message.data: element = message.data['element'] else: element = None msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key) print(msg) self.log.debug(msg) # Set configuration if new_value: msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, new_value) print msg self.log.debug(msg) self._set_config(techno, hostname, key, new_value) # Send configuration else: msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key) print(msg) self.log.debug(msg) if element: msg = "Request h=%s, t=%s, k=%s (send if element)" % ( hostname, techno, key) print(msg) self.log.debug(msg) self._send_config( techno, hostname, key, self._fetch_elmt_config(techno, element, key), element) else: msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname, techno, key) print(msg) self.log.debug(msg) if not key: msg = "Request h=%s, t=%s, k=%s (send if not key)" % ( hostname, techno, key) print(msg) self.log.debug(msg) keys = self._fetch_techno_config(techno, hostname, key).keys() values = self._fetch_techno_config(techno, hostname, key).values() self._send_config(techno, hostname, keys, values) else: msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % ( hostname, techno, key) print(msg) self.log.debug(msg) self._send_config( techno, hostname, key, self._fetch_techno_config(techno, hostname, key))
class DBConnector(XplPlugin, MQRep): ''' Manage the connection between database and the xPL stuff Should be the *only* object along with the StatsManager to access to the database on the core side ''' def __init__(self): ''' Initialize database and xPL connection ''' XplPlugin.__init__(self, 'dbmgr') MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug("Init database_manager instance") # Check for database connexion self._db = DbHelper() nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % ( nb_test, DATABASE_CONNECTION_NUM_TRY) print(msg) self.log.error(msg) msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT print(msg) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" print(msg) self.log.error(msg) self.force_leave() return msg = "Connected to the database" print(msg) self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error("Error while starting database engine : %s" % traceback.format_exc()) self.force_leave() return Listener(self._request_config_cb, self.myxpl, { 'schema': 'domogik.config', 'xpltype': 'xpl-cmnd' }) self.enable_hbeat() IOLoop.instance().start() def on_mdp_request(self, msg): if msg.get_action() == "config.get": plugin = msg._data['plugin'] hostname = msg._data['hostname'] key = msg._data['key'] if 'element' in msg._data.keys(): element = msg._data['element'] else: element = None if element: self._mdp_reply(plugin, hostname, key, self._fetch_elmt_config(plugin, element, key), element) elif not key: print 'TODO' else: self._mdp_reply( plugin, hostname, key, self._fetch_techno_config(plugin, hostname, key)) elif msg._action == "config.set": print 'TODO' def _mdp_reply(self, plugin, hostname, key, value, element=None): msg = MQMessage() msg.setaction('config.result') msg.add_data('plugin', plugin) msg.add_data('hostname', hostname) msg.add_data('key', key) msg.add_data('value', value) if element: msg.add_data('element', element) print msg.get() self.reply(msg.get()) def _request_config_cb(self, message): ''' Callback to receive a request for some config stuff @param message : the xPL message ''' #try: self._db = DbHelper(engine=self._engine) techno = message.data['plugin'] hostname = message.data['hostname'] key = message.data['key'] msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key) print(msg) self.log.debug(msg) if "value" in message.data: new_value = message.data['value'] else: new_value = None if "element" in message.data: element = message.data['element'] else: element = None msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key) print(msg) self.log.debug(msg) # Set configuration if new_value: msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, new_value) print msg self.log.debug(msg) self._set_config(techno, hostname, key, new_value) # Send configuration else: msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key) print(msg) self.log.debug(msg) if element: msg = "Request h=%s, t=%s, k=%s (send if element)" % ( hostname, techno, key) print(msg) self.log.debug(msg) self._send_config( techno, hostname, key, self._fetch_elmt_config(techno, element, key), element) else: msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname, techno, key) print(msg) self.log.debug(msg) if not key: msg = "Request h=%s, t=%s, k=%s (send if not key)" % ( hostname, techno, key) print(msg) self.log.debug(msg) keys = self._fetch_techno_config(techno, hostname, key).keys() values = self._fetch_techno_config(techno, hostname, key).values() self._send_config(techno, hostname, keys, values) else: msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % ( hostname, techno, key) print(msg) self.log.debug(msg) self._send_config( techno, hostname, key, self._fetch_techno_config(techno, hostname, key)) def _send_config(self, plugin, hostname, key, value, element=None): ''' Send a config value message for an element's config item @param plugin : the plugin of the element @param hostname : hostname @param element : the name of the element @param key : the key or list of keys of the config tuple(s) to fetch @param value : the value or list of values corresponding to the key(s) ''' msg = "Response h=%s, t=%s, k=%s, v=%s" % (hostname, plugin, key, value) print(msg) self.log.debug(msg) mess = XplMessage() mess.set_type('xpl-stat') mess.set_schema('domogik.config') mess.add_data({'plugin': plugin}) mess.add_data({'hostname': hostname}) if element: mess.add_data({'element': element}) # If key/value are lists, then we add a key=value for each item if isinstance(key, list): for (_key, _val) in zip(key, value): mess.add_data({_key: _val}) else: mess.add_data({key: value}) # mess.set_conf_key('target', plugin) self.myxpl.send(mess) def _fetch_elmt_config(self, techno, element, key): ''' Fetch an element's config value in the database @param techno : the plugin of the element @param element : the name of the element @param key : the key of the config tuple to fetch ''' #TODO : use the database vals = { 'x10': { 'a3': {}, 'a2': {}, } } return vals[techno][element][key] def _fetch_techno_config(self, techno, hostname, key): ''' Fetch a plugin global config value in the database @param techno : the plugin of the element @param hostname : hostname @param key : the key of the config tuple to fetch ''' # This array is here for information only but is not used anymore # Values are now on the database #vals = {'x10': {'heyu-cfg-path':'/etc/heyu/x10.conf', # 'heyu-file-0': 'TTY /dev/ttyUSB0', # 'heyu-file-1': 'TTY_AUX /dev/ttyUSB0 RFXCOM', # 'heyu-file-2': 'ALIAS back_door D5 DS10A 0x677'}, # 'global': {'pid-dir-path': '/var/run/'}, # 'onewire': {'temperature_refresh_delay' : '10'}, # 'cidmodem': {'device' : '/dev/ttyUSB1', # 'nbmaxtry' : '10', # 'interval' : '15'}, # 'mirror': {'device' : '/dev/hidraw0', # 'nbmaxtry' : '10', # 'interval' : '15'}, # 'xbmc_not': {'address' : '192.168.0.20:8080', # 'delay' : '15', # 'maxdelay' : '20'}, # 'gagenda': {'email' : "*****@*****.**", # 'password' : 'XXXXXXXX', # 'calendarname' : '*****@*****.**', # 'startup-plugin':'True'}, # 'teleinfo' : {'device' : '/dev/teleinfo', # 'interval' : '30'}, # 'dawndusk' : {'startup-plugin':'True'}, # 'plcbus' : {'device':'/dev/ttyUSB0'}, # } self.log.debug("FTC 1") try: if key: self.log.debug("FTC 2") try: self.log.debug("Get plg conf for %s / %s / %s" % (techno, hostname, key)) result = self._db.get_plugin_config(techno, hostname, key) # tricky loop as workaround for a (sqlalchemy?) bug : # sometimes the given result is for another plugin/key # so while we don't get the good data, we loop # This bug happens rarely while result.id != techno or \ result.hostname != hostname or \ result.key != key: self.log.debug("Bad result : %s/%s != %s/%s" % (result.id, result.key, plugin, key)) result = self._db.get_plugin_config( techno, hostname, key) self.log.debug("Get plg conf for %s / %s / %s Result=%s" % (techno, hostname, key, result)) val = result.value self.log.debug("Get plg conf for %s / %s / %s = %s" % (techno, hostname, key, val)) if val == '': val = "None" self.log.debug("Get plg conf for %s / %s / %s = %s (2)" % (techno, hostname, key, val)) return val except AttributeError: self.log.debug("Attribute error for %s / %s / %s" % (techno, hostname, key)) return "None" else: self.log.debug("FTC 3") vals = self._db.list_plugin_config(techno, hostname) res = {} for val in vals: if val == '': res[val.key] = "None" else: res[val.key] = val.value return res except: msg = "No config found h=%s, t=%s, k=%s" % (hostname, techno, key) print(msg) self.log.warn(msg) return "None" def _set_config(self, plugin, hostname, key, value): ''' Send a config value message for an element's config item @param plugin : the plugin of the element @param hostname : hostname @param key : the key to set @param value : the value to set ''' try: self._db.set_plugin_config(technology, hostname, key, value) mess = XplMessage() mess.set_type('xpl-stat') mess.set_schema('domogik.config') mess.add_data({'plugin': plugin}) mess.add_data({'hostname': hostname}) mess.add_data({'key': key}) mess.add_data({'value': value}) self.myxpl.send(mess) except: traceback.print_exc() msg = "Error while setting h=%s, t=%s, k=%s, v=%s" % ( hostname, techno, key, value) print(msg) self.log.warn(msg) return "None"
class XplManager(XplPlugin, MQAsyncSub): """ Statistics manager """ def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw', log_prefix = "") MQAsyncSub.__init__(self, self.zmq, 'xplgw', ['client.conversion', 'client.list']) self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') self.stats = None self.client_xpl_map = {} self.client_conversion_map = {} self._load_client_to_xpl_target() self._load_conversions() self.load() self.ready() def on_mdp_request(self, msg): # XplPlugin handles MQ Req/rep also try: XplPlugin.on_mdp_request(self, msg) if msg.get_action() == "reload": self.load() msg = MQMessage() msg.set_action( 'reload.result' ) self.reply(msg.get()) elif msg.get_action() == "cmd.send": self._send_xpl_command(msg) except: self.log.error(traceback.format_exc()) def on_message(self, msgid, content): try: if msgid == 'client.conversion': self._parse_conversions(content) elif msgid == 'client.list': self._parse_xpl_target(content) except: self.log.error(traceback.format_exc()) def _load_client_to_xpl_target(self): cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('client.list.get') response = cli.request('manager', msg.get(), timeout=10) if response: self._parse_xpl_target(response.get_data()) else: self.log.error(u"Updating client list was not successfull, no response from manager") def _parse_xpl_target(self, data): tmp = {} for cli in data: tmp[cli] = data[cli]['xpl_source'] self.client_xpl_map = tmp def _load_conversions(self): cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('client.conversion.get') response = cli.request('manager', msg.get(), timeout=10) if response: self._parse_conversions(response.get_data()) else: self.log.error(u"Updating client conversion list was not successfull, no response from manager") def _parse_conversions(self, data): tmp = {} for cli in data: tmp[cli] = data[cli] self.client_conversion_map = tmp def _send_xpl_command(self, data): """ Reply to config.get MQ req @param data : MQ req message Needed info in data: - cmdid => command id to send - cmdparams => key/value pair of all params needed for this command """ with self._db.session_scope(): self.log.info(u"Received new cmd request: {0}".format(data)) failed = False request = data.get_data() if 'cmdid' not in request: failed = "cmdid not in message data" if 'cmdparams' not in request: failed = "cmdparams not in message data" if not failed: # get the command cmd = self._db.get_command(request['cmdid']) if cmd is not None: if cmd.xpl_command is not None: xplcmd = cmd.xpl_command xplstat = self._db.get_xpl_stat(xplcmd.stat_id) if xplstat is not None: # get the device from the db dev = self._db.get_device(int(cmd.device_id)) msg = XplMessage() if not dev['client_id'] in self.client_xpl_map.keys(): self._load_client_to_xpl_target() if not dev['client_id'] in self.client_xpl_map.keys(): failed = "Can not fincd xpl source for {0} client_id".format(dev['client_id']) else: msg.set_target(self.client_xpl_map[dev['client_id']]) msg.set_source(self.myxpl.get_source()) msg.set_type("xpl-cmnd") msg.set_schema( xplcmd.schema) # static params for p in xplcmd.params: msg.add_data({p.key : p.value}) # dynamic params for p in cmd.params: if p.key in request['cmdparams']: value = request['cmdparams'][p.key] # chieck if we need a conversion if p.conversion is not None and p.conversion != '': if dev['client_id'] in self.client_conversion_map: if p.conversion in self.client_conversion_map[dev['client_id']]: exec(self.client_conversion_map[dev['client_id']][p.conversion]) value = locals()[p.conversion](value) msg.add_data({p.key : value}) else: failed = "Parameter ({0}) for device command msg is not provided in the mq message".format(p.key) if not failed: # send out the msg self.log.debug(u"sending xplmessage: {0}".format(msg)) self.myxpl.send(msg) ### Wait for answer stat_received = 0 if xplstat != None: # get xpl message from queue self.log.debug(u"Command : wait for answer...") sub = MQSyncSub( self.zmq, 'xplgw-command', ['device-stats'] ) stat = sub.wait_for_event() if stat is not None: reply = json.loads(stat['content']) reply_msg = MQMessage() reply_msg.set_action('cmd.send.result') reply_msg.add_data('stat', reply) reply_msg.add_data('status', True) reply_msg.add_data('reason', None) self.log.debug(u"mq reply".format(reply_msg.get())) self.reply(reply_msg.get()) if failed: self.log.error(failed) reply_msg = MQMessage() reply_msg.set_action('cmd.send.result') reply_msg.add_data('status', False) reply_msg.add_data('reason', failed) self.log.debug(u"mq reply".format(reply_msg.get())) self.reply(reply_msg.get()) def load(self): """ (re)load all xml files to (re)create _Stats objects """ self.log.info(u"Rest Stat Manager loading.... ") self._db.open_session() try: # not the first load : clean if self.stats != None: self.log.info(u"reloading") for stat in self.stats: self.myxpl.del_listener(stat.get_listener()) ### Load stats # key1, key2 = device_type_id, schema self.stats = [] created_stats = [] for stat in self._db.get_all_xpl_stat(): # xpl-trig self.stats.append(self._Stat(self.myxpl, stat, "xpl-trig", \ self.log, self.pub, self.client_conversion_map)) # xpl-stat self.stats.append(self._Stat(self.myxpl, stat, "xpl-stat", \ self.log, self.pub, self.client_conversion_map)) except: self.log.error(u"%s" % traceback.format_exc()) self._db.close_session() self.log.info(u"Loading finished") class _Stat: """ This class define a statistic parser and logger instance Each instance create a Listener and the associated callbacks """ def __init__(self, xpl, stat, xpl_type, log, pub, conversions): """ Initialize a stat instance @param xpl : A xpl manager instance @param stat : A XplStat reference @param xpl-type: what xpl-type to listen for """ ### Rest data self._log_stats = log self._stat = stat self._pub = pub self._conv = conversions ### build the filter params = {'schema': stat.schema, 'xpltype': xpl_type} for param in stat.params: if param.static: params[param.key] = param.value ### start the listener self._log_stats.info("creating listener for %s" % (params)) self._listener = Listener(self._callback, xpl, params) def get_listener(self): """ getter for lsitener object """ return self._listener def _callback(self, message): """ Callback for the xpl message @param message : the Xpl message received """ self._log_stats.debug( "_callback started for: {0}".format(message) ) db = DbHelper() current_date = calendar.timegm(time.gmtime()) stored_value = None try: # find what parameter to store for param in self._stat.params: # self._log_stats.debug("Checking param {0}".format(param)) if param.sensor_id is not None and param.static is False: if param.key in message.data: with db.session_scope(): value = message.data[param.key] # self._log_stats.debug( \ # "Key found {0} with value {1}." \ # .format(param.key, value)) store = True if param.ignore_values: if value in eval(param.ignore_values): self._log_stats.debug( \ "Value {0} is in the ignore list {0}, so not storing." \ .format(value, param.ignore_values)) store = False if store: # get the sensor and dev sen = db.get_sensor(param.sensor_id) dev = db.get_device(sen.device_id) # check if we need a conversion if sen.conversion is not None and sen.conversion != '': if dev['client_id'] in self._conv: if sen.conversion in self._conv[dev['client_id']]: self._log_stats.debug( \ "Calling conversion {0}".format(sen.conversion)) exec(self._conv[dev['client_id']][sen.conversion]) value = locals()[sen.conversion](value) self._log_stats.info( \ "Storing stat for device '{0}' ({1}) and sensor'{2}' ({3}): key '{4}' with value '{5}' after conversion." \ .format(dev['name'], dev['id'], sen.name, sen.id, param.key, value)) # do the store stored_value = value try: db.add_sensor_history(\ param.sensor_id, \ value, \ current_date) except: self._log_stats.error("Error when adding sensor history : {0}".format(traceback.format_exc())) else: self._log_stats.debug("Don't need to store this value") # publish the result self._pub.send_event('device-stats', \ {"timestamp" : current_date, \ "device_id" : dev['id'], \ "sensor_id" : sen.id, \ "stored_value" : stored_value}) #else: # self._log_stats.debug("Key not found in message data") #else: # self._log_stats.debug("No sensor attached") except: self._log_stats.error(traceback.format_exc())
def _callback(self, message): """ Callback for the xpl message @param message : the Xpl message received """ self._log_stats.debug( "_callback started for: {0}".format(message) ) db = DbHelper() current_date = calendar.timegm(time.gmtime()) stored_value = None try: # find what parameter to store for param in self._stat.params: # self._log_stats.debug("Checking param {0}".format(param)) if param.sensor_id is not None and param.static is False: if param.key in message.data: with db.session_scope(): value = message.data[param.key] # self._log_stats.debug( \ # "Key found {0} with value {1}." \ # .format(param.key, value)) store = True if param.ignore_values: if value in eval(param.ignore_values): self._log_stats.debug( \ "Value {0} is in the ignore list {0}, so not storing." \ .format(value, param.ignore_values)) store = False if store: # get the sensor and dev sen = db.get_sensor(param.sensor_id) dev = db.get_device(sen.device_id) # check if we need a conversion if sen.conversion is not None and sen.conversion != '': if dev['client_id'] in self._conv: if sen.conversion in self._conv[dev['client_id']]: self._log_stats.debug( \ "Calling conversion {0}".format(sen.conversion)) exec(self._conv[dev['client_id']][sen.conversion]) value = locals()[sen.conversion](value) self._log_stats.info( \ "Storing stat for device '{0}' ({1}) and sensor'{2}' ({3}): key '{4}' with value '{5}' after conversion." \ .format(dev['name'], dev['id'], sen.name, sen.id, param.key, value)) # do the store stored_value = value try: db.add_sensor_history(\ param.sensor_id, \ value, \ current_date) except: self._log_stats.error("Error when adding sensor history : {0}".format(traceback.format_exc())) else: self._log_stats.debug("Don't need to store this value") # publish the result self._pub.send_event('device-stats', \ {"timestamp" : current_date, \ "device_id" : dev['id'], \ "sensor_id" : sen.id, \ "stored_value" : stored_value}) #else: # self._log_stats.debug("Key not found in message data") #else: # self._log_stats.debug("No sensor attached") except: self._log_stats.error(traceback.format_exc())
class PackageData(): """ Tool to insert necessary data in database """ def __init__(self, json_path): """ Init tool @param plugin_name : plugin name """ self._db = DbHelper() try: self.pkg = PackageJson(path = json_path).json except PackageException as exp: print("Error in json file:") print( exp.value ) exit() except: print(str(traceback.format_exc())) return print("Json file OK") # check type == plugin if self.pkg["identity"]["type"] not in ["plugin", "external"]: print("Error : this package type is not recognized") exit() # check if json version is at least 2 if self.pkg['json_version'] < 2: print("Error : this package is to old for this version of domogik") exit() def insert(self): """ Insert data for plugin """ ### Plugin print("plugin %s" % self.pkg["identity"]["id"]) if self._db.get_plugin(self.pkg["identity"]["id"]) == None: # add if not exists print("add...") self._db.add_plugin(self.pkg["identity"]["id"], self.pkg["identity"]["description"], self.pkg["identity"]["version"]) else: # update if exists print("update...") self._db.update_plugin(self.pkg["identity"]["id"], self.pkg["identity"]["description"], self.pkg["identity"]["version"]) ### Device types for device_type in self.pkg["device_types"].keys(): print("Device type %s" % device_type) device_type = self.pkg["device_types"][device_type] if self._db.get_device_type_by_id(device_type["id"]) == None: # add if not exists print("add...") self._db.add_device_type(device_type["id"], device_type["name"], self.pkg["identity"]["id"], device_type["description"]) else: # update if exists print("update...") self._db.update_device_type(device_type["id"], device_type["name"], self.pkg["identity"]["id"], device_type["description"])
class StatsManager: """ Listen on the xPL network and keep stats of device and system state """ def __init__(self, handler_params, xpl): """ @param handler_params : The server params @param xpl : A xPL Manager instance """ try: self.myxpl = xpl # logging initialization log = logger.Logger('rest-stat') self._log_stats = log.get_logger('rest-stat') self._log_stats.info("Rest Stat Manager initialisation...") # create the dbHelper self._db = DbHelper() ### Rest data self.handler_params = handler_params self.handler_params.append(self._log_stats) self.handler_params.append(self._db) self._event_requests = self.handler_params[0]._event_requests self.get_exception = self.handler_params[0].get_exception self.stats = None except : self._log_stats.error("%s" % traceback.format_exc()) def load(self): """ (re)load all xml files to (re)create _Stats objects """ self._log_stats.info("Rest Stat Manager loading.... ") try: # not the first load : clean if self.stats != None: for x in self.stats: self.myxpl.del_listener(x.get_listener()) ### Load stats # key1, key2 = device_type_id, schema self.stats = [] for sen in self._db.get_all_sensor(): self._log_stats.debug(sen) statparam = self._db.get_xpl_stat_param_by_sensor(sen.id) if statparam is None: self._log_stats.error('Corresponding xpl-stat param can not be found for sensor %s' % (sen)) continue stat = self._db.get_xpl_stat(statparam.xplstat_id) if stat is None: self._log_stats.error('Corresponding xpl-stat can not be found for xplstatparam %s' % (statparam)) continue dev = self._db.get_device(stat.device_id) if dev is None: self._log_stats.error('Corresponding device can not be found for xpl-stat %s' % (stat)) continue # xpl-trig self.stats.append(self._Stat(self.myxpl, dev, stat, sen, \ "xpl-trig", self._log_stats, \ self._log_stats, self._db, \ self._event_requests)) # xpl-stat self.stats.append(self._Stat(self.myxpl, dev, stat, sen, \ "xpl-stat", self._log_stats, \ self._log_stats, self._db, \ self._event_requests)) except: self._log_stats.error("%s" % traceback.format_exc()) class _Stat: """ This class define a statistic parser and logger instance Each instance create a Listener and the associated callbacks """ def __init__(self, xpl, dev, stat, sensor, xpl_type, log_stats, log_stats_unknown, db, event_requests): """ Initialize a stat instance @param xpl : A xpl manager instance @param dev : A Device reference @param stat : A XplStat reference @param sensor: A Sensor reference @param xpl-type: what xpl-type to listen for """ ### Rest data self._event_requests = event_requests self._db = db self._log_stats = log_stats #self._log_stats_unknown = log_stats_unknown self._dev = dev self._stat = stat self._sen = sensor ### build the filter params = {'schema': stat.schema, 'xpltype': xpl_type} for p in stat.params: if p.static: params[p.key] = p.value ### start the listener self._log_stats.debug("creating listener for %s" % (params)) self._listener = Listener(self._callback, xpl, params) def get_listener(self): """ getter for lsitener object """ return self._listener def _callback(self, message): """ Callback for the xpl message @param message : the Xpl message received """ my_db = DbHelper() self._log_stats.debug("Stat received for device %s." \ % (self._dev.name)) current_date = calendar.timegm(time.gmtime()) device_data = [] try: # find what parameter to store for p in self._stat.params: if p.sensor_id is not None: if p.key in message.data: value = message.data[p.key] self._log_stats.debug("Key found %s with value %s." \ % (p.key, value)) store = True if p.ignore_values: if value in eval(p.ignore_values): self._log_stats.debug("Value %s is in the ignore list %s, so not storing." \ % (value, p.ignore_values)) store = False if store: # check if we need a conversion if self._sen.conversion is not None and self._sen.conversion != '': value = call_package_conversion(\ self._log_stats, self._dev.device_type.plugin_id, \ self._sen.conversion, value) self._log_stats.debug("Key found %s with value %s after conversion." \ % (p.key, value)) # do the store device_data.append({"value" : value, "sensor": p.sensor_id}) my_db.add_sensor_history(p.sensor_id, value, current_date) except: error = "Error when processing stat : %s" % traceback.format_exc() print("==== Error in Stats ====") print(error) print("========================") self._log_stats.error(error) # put data in the event queue self._event_requests.add_in_queues(self._dev.id, {"timestamp" : current_date, "device_id" : self._dev.id, "data" : device_data}) del(my_db)
class DBConnector(Plugin, MQRep): ''' Manage the connection between database and the plugins Should be the *only* object along with the StatsManager to access to the database on the core side ''' def __init__(self): ''' Initialize database and xPL connection ''' Plugin.__init__(self, 'dbmgr', log_prefix='core_') # Already done in Plugin #MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug(u"Init database_manager instance") # Check for database connexion self._db = DbHelper() with self._db.session_scope(): # TODO : move in a function and use it (also used in dbmgr) nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc()) self.log.error(msg) msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return msg = "Connected to the database" self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc())) self.force_leave() return self.ready() # Already done in ready() #IOLoop.instance().start() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ try: with self._db.session_scope(): # Plugin handles MQ Req/rep also Plugin.on_mdp_request(self, msg) # configuration if msg.get_action() == "config.get": self._mdp_reply_config_get(msg) elif msg.get_action() == "config.set": self._mdp_reply_config_set(msg) elif msg.get_action() == "config.delete": self._mdp_reply_config_delete(msg) # devices list elif msg.get_action() == "device.get": self._mdp_reply_devices_result(msg) # device get params elif msg.get_action() == "device.params": self._mdp_reply_devices_params_result(msg) # device create elif msg.get_action() == "device.create": self._mdp_reply_devices_create_result(msg) # device delete elif msg.get_action() == "device.delete": self._mdp_reply_devices_delete_result(msg) # device update elif msg.get_action() == "device.update": self._mdp_reply_devices_update_result(msg) # deviceparam update elif msg.get_action() == "deviceparam.update": self._mdp_reply_deviceparam_update_result(msg) # sensor update elif msg.get_action() == "sensor.update": self._mdp_reply_sensor_update_result(msg) # sensor history elif msg.get_action() == "sensor_history.get": self._mdp_reply_sensor_history(msg) except: msg = "Error while processing request. Message is : {0}. Error is : {1}".format(msg, traceback.format_exc()) self.log.error(msg) def _mdp_reply_config_get(self, data): """ Reply to config.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] not in ["plugin", "brain", "interface"]: status = False reason = "Configuration request not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if 'key' not in msg_data: get_all_keys = True key = "*" else: get_all_keys = False key = msg_data['key'] if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('key', key) # we let this here to display key or * depending on the case try: if get_all_keys == True: config = self._db.list_plugin_config(type, name, host) self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, config)) json_config = {} for elt in config: json_config[elt.key] = self.convert(elt.value) msg.add_data('data', json_config) else: value = self._fetch_techno_config(name, type, host, key) # temporary fix : should be done in a better way (on db side) value = self.convert(value) self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, value)) msg.add_data('value', value) except: status = False reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(type, name, host, key, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_set(self, data): """ Reply to config.set MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config set : missing 'type' field : {0}".format(data) if msg_data['type'] not in ["plugin", "brain", "interface"]: status = False reason = "You are not able to configure items for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config set : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config set : missing 'host' field : {0}".format(data) if 'data' not in msg_data: status = False reason = "Config set : missing 'data' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] data = msg_data['data'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: # we add a configured key set to true to tell the UIs and plugins that there are some configuration elements self._db.set_plugin_config(type, name, host, "configured", True) for key in msg_data['data']: self._db.set_plugin_config(type, name, host, key, data[key]) self.publish_config_updated(type, name, host) except: reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(type, name, host, traceback.format_exc()) status = False self.log.error(reason) msg.add_data('status', status) msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_delete(self, data): """ Reply to config.delete MQ req Delete all the config items for the given type, name and host @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] not in ["plugin", "brain", "interface"]: status = False reason = "Configuration deletion not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: self._db.del_plugin_config(type, name, host) self.log.info(u"Delete config for {0} {1}".format(type, name)) self.publish_config_updated(type, name, host) except: status = False reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(type, name, host, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _fetch_techno_config(self, name, type, host, key): ''' Fetch a plugin global config value in the database @param name : the plugin of the element @param host : hostname @param key : the key of the config tuple to fetch ''' try: try: result = self._db.get_plugin_config(type, name, host, key) # tricky loop as workaround for a (sqlalchemy?) bug : # sometimes the given result is for another plugin/key # so while we don't get the good data, we loop # This bug happens rarely while result.id != name or \ result.type != type or \ result.hostname != host or \ result.key != key: self.log.debug(u"Bad result : {0}-{1}/{2} != {3}/{4}".format(result.id, result.type, result.key, plugin, key)) result = self._db.get_plugin_config(type, name, host, key) val = result.value if val == '': val = "None" return val except AttributeError: # if no result is found #self.log.error(u"Attribute error : {0}".format(traceback.format_exc())) return "None" except: msg = "No config found host={0}, plugin={1}, key={2}".format(host, name, key) self.log.warn(msg) return "None" def _mdp_reply_devices_delete_result(self, data): status = True reason = False self.log.debug(u"Deleting device : {0}".format(data)) try: did = data.get_data()['did'] if did: res = self._db.del_device(did) if not res: status = False else: status = True else: status = False reason = "There is no such device" self.log.debug(reason) # delete done except DbHelperException as d: status = False reason = "Error while deleting device: {0}".format(d.value) self.log.error(reason) except: status = False reason = "Error while deleting device: {0}".format(traceback.format_exc()) self.log.error(reason) # send the result msg = MQMessage() msg.set_action('device.delete.result') msg.add_data('status', status) if reason: msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) # send the pub message if status and res: self._pub.send_event('device.update', {"device_id" : did, "client_id" : res.client_id}) def _mdp_reply_sensor_update_result(self, data): status = True reason = False self.log.debug(u"Updating sensor : {0}".format(data)) try: data = data.get_data() if 'sid' in data: sid = data['sid'] if 'history_round' not in data: hround = None else: hround = data['history_round'] if 'history_store' not in data: hstore = None else: hstore = data['history_store'] if 'history_max' not in data: hmax = None else: hmax = data['history_max'] if 'history_expire' not in data: hexpire = None else: hexpire = data['history_expire'] if 'timeout' not in data: timeout = None else: timeout = data['timeout'] if 'formula' not in data: formula = None else: formula = data['formula'] if 'data_type' not in data: data_type = None else: data_type = data['data_type'] # do the update res = self._db.update_sensor(sid, \ history_round=hround, \ history_store=hstore, \ history_max=hmax, \ history_expire=hexpire, \ timeout=timeout, \ formula=formula, \ data_type=data_type) if not res: status = False else: status = True else: status = False reason = "There is no such sensor" self.log.debug(reason) # delete done except DbHelperException as d: status = False reason = "Error while updating sensor: {0}".format(d.value) self.log.error(reason) except: status = False reason = "Error while updating sensor: {0}".format(traceback.format_exc()) self.log.error(reason) # send the result msg = MQMessage() msg.set_action('sensor.update.result') msg.add_data('status', status) if reason: msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) # send the pub message if status and res: dev = self._db.get_device(res.device_id) self._pub.send_event('device.update', {"device_id" : res.device_id, "client_id" : dev['client_id']}) def _mdp_reply_deviceparam_update_result(self, data): status = True reason = False self.log.debug(u"Updating device param : {0}".format(data)) try: data = data.get_data() if 'dpid' in data: dpid = data['dpid'] val = data['value'] # do the update res = self._db.udpate_device_param(dpid, value=val) if not res: status = False else: status = True else: status = False reason = "There is no such device param" self.log.debug(reason) # delete done except DbHelperException as d: status = False reason = "Error while updating device param: {0}".format(d.value) self.log.error(reason) except: status = False reason = "Error while updating device param: {0}".format(traceback.format_exc()) self.log.error(reason) # send the result msg = MQMessage() msg.set_action('deviceparam.update.result') msg.add_data('status', status) if reason: msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) # send the pub message if status and res: dev = self._db.get_device(res.device_id) self._pub.send_event('device.update', {"device_id" : res.device_id, "client_id" : dev['client_id']}) def _mdp_reply_devices_update_result(self, data): status = True reason = False self.log.debug(u"Updating device : {0}".format(data)) try: data = data.get_data() if 'did' in data: did = data['did'] if 'name' not in data: name = None else: name = data['name'] if 'reference' not in data: ref = None else: ref = data['reference'] if 'description' not in data: desc = None else: desc = data['description'] # do the update res = self._db.update_device(did, \ d_name=name, \ d_description=desc, \ d_reference=ref) if not res: status = False else: status = True else: status = False reason = "There is no such device" self.log.debug(reason) # delete done except DbHelperException as d: status = False reason = "Error while updating device: {0}".format(d.value) self.log.error(reason) except: status = False reason = "Error while updating device: {0}".format(traceback.format_exc()) self.log.error(reason) # send the result msg = MQMessage() msg.set_action('device.update.result') msg.add_data('status', status) if reason: msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) # send the pub message if status and res: self._pub.send_event('device.update', {"device_id" : res.id, "client_id" : res.client_id}) def _mdp_reply_devices_create_result(self, data): status = True reason = False result = False # get the filled package json params = data.get_data()['data'] # get the json cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('device_types.get') msg.add_data('device_type', params['device_type']) res = cli.request('manager', msg.get(), timeout=10) del cli if res is None: status = False reason = "Manager is not replying to the mq request" pjson = res.get_data() if pjson is None: status = False reason = "No data for {0} found by manager".format(params['device_type']) pjson = pjson[params['device_type']] if pjson is None: status = False reason = "The json for {0} found by manager is empty".format(params['device_type']) if status: # call the add device function res = self._db.add_full_device(params, pjson) if not res: status = False reason = "An error occured while adding the device in database. Please check the file dbmgr.log for more informations" else: status = True reason = False result = res msg = MQMessage() msg.set_action('device.create.result') if reason: msg.add_data('reason', reason) if result: msg.add_data('result', result) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) # send the pub message if status and res: self._pub.send_event('device.update', {"device_id" : res['id'], "client_id" : res['client_id']}) def _mdp_reply_devices_params_result(self, data): """ Reply to device.params mq req @param data : MQ req message => should contain - device_type """ status = True try: # check we have all the needed info msg_data = data.get_data() if 'device_type' not in msg_data: status = False reason = "Device params request : missing 'cevice_type' field : {0}".format(data) else: dev_type_id = msg_data['device_type'] # check the received info if status: cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('device_types.get') msg.add_data('device_type', dev_type_id) res = cli.request('manager', msg.get(), timeout=10) del cli if res is None: status = False reason = "Manager is not replying to the mq request" if status: pjson = res.get_data() if pjson is None: status = False reason = "No data for {0} found by manager".format(msg_data['device_type']) if status: pjson = pjson[dev_type_id] if pjson is None: status = False reason = "The json for {0} found by manager is empty".format(msg_data['device_type']) self.log.debug("Device Params result : json received by the manager is : {0}".format(pjson)) if not status: # we don't have all info so exit msg = MQMessage() msg.set_action('device.params.result') msg.add_data('result', 'Failed') msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) return # we have the json now, build the params msg = MQMessage() msg.set_action('device.params.result') stats = [] result = {} result['device_type'] = dev_type_id result['name'] = "" result['reference'] = "" result['description'] = "" # append the global xpl and on-xpl params result['xpl'] = [] result['global'] = [] for param in pjson['device_types'][dev_type_id]['parameters']: if param['xpl']: del param['xpl'] result['xpl'].append(param) else: del param['xpl'] result['global'].append(param) # find the xplCommands result['xpl_commands'] = {} for cmdn in pjson['device_types'][dev_type_id]['commands']: cmd = pjson['commands'][cmdn] if 'xpl_command'in cmd: xcmdn = cmd['xpl_command'] xcmd = pjson['xpl_commands'][xcmdn] result['xpl_commands'][xcmdn] = [] stats.append( xcmd['xplstat_name'] ) for param in xcmd['parameters']['device']: result['xpl_commands'][xcmdn].append(param) # find the xplStats sensors = pjson['device_types'][dev_type_id]['sensors'] #print("SENSORS = {0}".format(sensors)) for xstatn in pjson['xpl_stats']: #print("XSTATN = {0}".format(xstatn)) xstat = pjson['xpl_stats'][xstatn] for sparam in xstat['parameters']['dynamic']: #print("XSTATN = {0}, SPARAM = {1}".format(xstatn, sparam)) #if 'sensor' in sparam and xstatn in sensors: # => This condition was used to fix a bug which occurs while creating complexe devices for rfxcom # But is introduced a bug for the geoloc plugin... # In fact we had to fix the rfxcom info.json file (open_close uses now rssi_open_close instead of # rssi_lighting2 # So, this one is NOT the good one. if 'sensor' in sparam: # => this condition was the original one restored to make the geoloc pluin ok for tests # Strangely, there is no issue while using the admin (which uses only mq) # but is sucks with test library which uses rest... # This one is the good one if sparam['sensor'] in sensors: #print("ADD") stats.append(xstatn) result['xpl_stats'] = {} #print("STATS = {0}".format(stats)) for xstatn in stats: xstat = pjson['xpl_stats'][xstatn] result['xpl_stats'][xstatn] = [] for param in xstat['parameters']['device']: result['xpl_stats'][xstatn].append(param) # return the data msg.add_data('result', result) self.log.debug(msg.get()) self.reply(msg.get()) except: self.log.error("Error when replying to device.params for data={0}. Error: {1}".format(data, traceback.format_exc())) def _mdp_reply_devices_result(self, data): """ Reply to device.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('device.result') status = True msg_data = data.get_data() # request for all devices if 'type' not in msg_data and \ 'name' not in msg_data and \ 'host' not in msg_data and \ 'timestamp' not in msg_data: reason = "" status = True dev_list = self._db.list_devices() dev_json = dev_list msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('devices', dev_json) elif 'timestamp'in msg_data: # request for all devices that changed after timestamp reason = "" status = True dev_list = self._db.list_devices_by_timestamp(msg_data['timestamp']) dev_json = dev_list msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('devices', dev_json) else: # request for all devices of one client if 'type' not in msg_data: status = False reason = "Devices request : missing 'type' field : {0}".format(data) if 'name' not in msg_data: status = False reason = "Devices request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Devices request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(type, name, host)) #dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False), dev_json = dev_list msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('devices', dev_json) self.reply(msg.get()) def _mdp_reply_sensor_history(self, data): """ Reply to sensor_history.get MQ req @param data : MQ req message If no other param than the sensor id, return the last value """ msg = MQMessage() msg.set_action('sensor_history.result') status = True reason = "" ### process parameters msg_data = data.get_data() if 'sensor_id' in msg_data: sensor_id = msg_data['sensor_id'] else: reason = "ERROR when getting sensor history. No sensor_id declared in the message" self.log.error(reason) status = False sensor_id = None if 'mode' in msg_data: if msg_data['mode'] == "last": mode = "last" elif msg_data['mode'] == "period": mode = "period" else: reason = "ERROR when getting sensor history. No valid type (last, from) declared in the message" self.log.error(reason) status = False mode = None else: reason = "ERROR when getting sensor history. No type (last, from) declared in the message" self.log.error(reason) status = False sensor_id = None values = None ### last N values if mode == "last": if 'number' in msg_data: number = msg_data['number'] else: number = 1 try: history = self._db.list_sensor_history(sensor_id, number) if len(history) == 0: #values = None values = self._db.get_last_sensor_value(sensor_id) else: values = self._db.list_sensor_history(sensor_id, number) except: self.log.error("ERROR when getting sensor history for id = {0} : {1}".format(sensor_id, traceback.format_exc())) reason = "ERROR : {0}".format(traceback.format_exc()) status = False ### period elif mode == "period": if 'from' in msg_data: frm = msg_data['from'] else: reason = "ERROR when getting sensor history. No key 'from' defined for mode = 'period'!" self.log.error(reason) status = False frm = None if 'to' in msg_data: to = msg_data['to'] else: to = None if frm != None and to == None: values = self._db.list_sensor_history_between(sensor_id, frm) print(values) else: # TODO values = "TODO" msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('sensor_id', sensor_id) msg.add_data('mode', mode) msg.add_data('values', values) self.reply(msg.get()) def convert(self, data): """ Do some conversions on data """ if data == "True": data = True if data == "False": data = False return data def publish_config_updated(self, type, name, host): """ Publish over the MQ a message to inform that a plugin configuration has been updated @param type : package type (plugin) @param name : package name @param host : host """ self.log.debug("Publish configuration update notification for {0}-{1}.{2}".format(type, name, host)) self._pub.send_event('plugin.configuration', {"type" : type, "name" : name, "host" : host, "event" : "updated"})
def check_args(argv): """Check arguments passed to the program""" def usage(prog_name): """Print program usage""" print("Usage : %s [-s [all|minute[,hour][,day][,week][,month[,year]]] [-I]" % prog_name) print("-s, --statistics=STATS_LIST\tSTATS_LIST can be : all or minute,hours,day,week,month,year") print("-I, --noinsert\t\t\tUse existing data in the database") if __name__ == "__main__": stats_filter = False try: opts, args = getopt.getopt(sys.argv[1:], "hs:I", ["help", "stats=", "noinsert"]) _db = DbHelper(use_test_db=True) print("Using %s database" % _db.get_db_type()) for opt, arg_list in opts: if opt in ("-s", "--stats"): possible_args = ['all', 'minute', 'hour', 'day', 'week', 'month', 'year'] if arg_list is not None: filter_list = arg_list.split(",") # Check args are ok for -s option for p_filter in filter_list: if p_filter not in filter_list: print("Wrong argument for statistics, must be one of : %s" % (",".join(possible_args))) usage(sys.argv[0]) if 'all' in filter_list: filter_list = possible_args[1:] stats_filter = True
class DBConnector(Plugin, MQRep): ''' Manage the connection between database and the plugins Should be the *only* object along with the StatsManager to access to the database on the core side ''' def __init__(self): ''' Initialize database and xPL connection ''' Plugin.__init__(self, 'dbmgr') # Already done in Plugin #MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug(u"Init database_manager instance") # Check for database connexion self._db = DbHelper() with self._db.session_scope(): nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}".format(nb_test, DATABASE_CONNECTION_NUM_TRY) self.log.error(msg) msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return msg = "Connected to the database" self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc())) self.force_leave() return self.ready() # Already done in ready() #IOLoop.instance().start() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ try: with self._db.session_scope(): # Plugin handles MQ Req/rep also Plugin.on_mdp_request(self, msg) # configuration if msg.get_action() == "config.get": self._mdp_reply_config_get(msg) elif msg.get_action() == "config.set": self._mdp_reply_config_set(msg) elif msg.get_action() == "config.delete": self._mdp_reply_config_delete(msg) # devices list elif msg.get_action() == "device.get": self._mdp_reply_devices_result(msg) # device get params elif msg.get_action() == "device.params": self._mdp_reply_devices_params_result(msg) # device create elif msg.get_action() == "device.create": self._mdp_reply_devices_create_result(msg) # device delete elif msg.get_action() == "device.delete": self._mdp_reply_devices_delete_result(msg) except: msg = "Error while processing request. Message is : {0}. Error is : {1}".format(msg, traceback.format_exc()) self.log.error(msg) def _mdp_reply_config_get(self, data): """ Reply to config.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config request not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if 'key' not in msg_data: get_all_keys = True key = "*" else: get_all_keys = False key = msg_data['key'] if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('key', key) # we let this here to display key or * depending on the case try: if get_all_keys == True: config = self._db.list_plugin_config(name, host) self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, config)) json_config = {} for elt in config: json_config[elt.key] = self.convert(elt.value) msg.add_data('data', json_config) else: value = self._fetch_techno_config(name, host, key) # temporary fix : should be done in a better way (on db side) value = self.convert(value) self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, value)) msg.add_data('value', value) except: status = False reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(type, name, host, key, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_set(self, data): """ Reply to config.set MQ req @param data : MQ req message """ print "#################" msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config set : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config set not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config set : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config set : missing 'host' field : {0}".format(data) if 'data' not in msg_data: status = False reason = "Config set : missing 'data' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] data = msg_data['data'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: # we add a configured key set to true to tell the UIs and plugins that there are some configuration elements self._db.set_plugin_config(name, host, "configured", True) for key in msg_data['data']: self._db.set_plugin_config(name, host, key, data[key]) self.publish_config_updated(type, name, host) except: reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(type, name, host, traceback.format_exc()) status = False self.log.error(reason) msg.add_data('status', status) msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_delete(self, data): """ Reply to config.delete MQ req Delete all the config items for the given type, name and host @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config request not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: self._db.del_plugin_config(name, host) self.log.info(u"Delete config for {0} {1}".format(type, name)) self.publish_config_updated(type, name, host) except: status = False reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(type, name, host, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _fetch_techno_config(self, name, host, key): ''' Fetch a plugin global config value in the database @param name : the plugin of the element @param host : hostname @param key : the key of the config tuple to fetch ''' try: try: result = self._db.get_plugin_config(name, host, key) # tricky loop as workaround for a (sqlalchemy?) bug : # sometimes the given result is for another plugin/key # so while we don't get the good data, we loop # This bug happens rarely while result.id != name or \ result.hostname != host or \ result.key != key: self.log.debug(u"Bad result : {0}/{1} != {2}/{3}".format(result.id, result.key, plugin, key)) result = self._db.get_plugin_config(name, host, key) val = result.value if val == '': val = "None" return val except AttributeError: # if no result is found #self.log.error(u"Attribute error : {0}".format(traceback.format_exc())) return "None" except: msg = "No config found host={0}, plugin={1}, key={2}".format(host, name, key) self.log.warn(msg) return "None" def _mdp_reply_devices_delete_result(self, data): status = True reason = False try: did = data.get_data()['did'] if did: res = self._db.del_device(did) if not res: status = False else: status = True else: status = False reason = "Device delete failed" # delete done self.reload_stats() except DbHelperException as d: status = False reason = "Error while deleting device: {0}".format(d.value) except: status = False reason = "Error while deleting device: {0}".format(traceback.format_exc()) # send the result msg = MQMessage() msg.set_action('device.delete.result') msg.add_data('status', status) if reason: msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_devices_create_result(self, data): status = True reason = False result = False # get the filled package json params = data.get_data()['data'] # get the json cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('device_types.get') msg.add_data('device_type', params['device_type']) res = cli.request('manager', msg.get(), timeout=10) del cli if res is None: status = False reason = "Manager is not replying to the mq request" pjson = res.get_data() if pjson is None: status = False reason = "No data for {0} found by manager".format(params['device_type']) pjson = pjson[params['device_type']] if pjson is None: status = False reason = "The json for {0} found by manager is empty".format(params['device_type']) if status: # call the add device function res = self._db.add_full_device(params, pjson) if not res: status = False reason = "An error occured while adding the device in database. Please check the file dbmgr.log for more informations" else: status = True reason = False result = res msg = MQMessage() msg.set_action('device.create.result') if reason: msg.add_data('reason', reason) if result: msg.add_data('result', result) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_devices_params_result(self, data): """ Reply to device.params mq req @param data : MQ req message => should contain - device_type """ status = True # check we have all the needed info msg_data = data.get_data() if 'device_type' not in msg_data: status = False reason = "Device params request : missing 'cevice_type' field : {0}".format(data) else: dev_type_id = msg_data['device_type'] # check the received info if status: cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('device_types.get') msg.add_data('device_type', dev_type_id) res = cli.request('manager', msg.get(), timeout=10) del cli if res is None: status = False reason = "Manager is not replying to the mq request" pjson = res.get_data() if pjson is None: status = False reason = "No data for {0} found by manager".format(msg_data['device_type']) pjson = pjson[dev_type_id] if pjson is None: status = False reason = "The json for {0} found by manager is empty".format(msg_data['device_type']) # we have the json now, build the params msg = MQMessage() msg.set_action('device.params.result') stats = [] result = {} result['device_type'] = dev_type_id result['name'] = "" result['reference'] = "" result['description'] = "" # append the global xpl and on-xpl params result['xpl'] = [] result['global'] = [] for param in pjson['device_types'][dev_type_id]['parameters']: if param['xpl']: del param['xpl'] result['xpl'].append(param) else: del param['xpl'] result['global'].append(param) # find the xplCommands result['xpl_commands'] = {} for cmdn in pjson['device_types'][dev_type_id]['commands']: cmd = pjson['commands'][cmdn] if 'xpl_command'in cmd: xcmdn = cmd['xpl_command'] xcmd = pjson['xpl_commands'][xcmdn] result['xpl_commands'][xcmdn] = [] stats.append( xcmd['xplstat_name'] ) for param in xcmd['parameters']['device']: result['xpl_commands'][xcmdn].append(param) # find the xplStats sensors = pjson['device_types'][dev_type_id]['sensors'] for xstatn in pjson['xpl_stats']: xstat = pjson['xpl_stats'][xstatn] if xstat['parameters']['dynamic'] in sensors: stats.append(xstatn) result['xpl_stats'] = {} for xstatn in stats: xtat = pjson['xpl_stats'][xstatn] result['xpl_stats'][xstatn] = [] for param in xstat['parameters']['device']: result['xpl_stats'][xstatn].append(param) # return the data msg.add_data('result', result) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_devices_result(self, data): """ Reply to device.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('device.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Devices request : missing 'type' field : {0}".format(data) if 'name' not in msg_data: status = False reason = "Devices request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Devices request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] #if type == "plugin": # type = DMG_VENDOR_ID name = msg_data['name'] host = msg_data['host'] dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(type, name, host)) #dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False), dev_json = dev_list print(dev_json) msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('devices', dev_json) self.reply(msg.get()) def convert(self, data): """ Do some conversions on data """ if data == "True": data = True if data == "False": data = False return data def publish_config_updated(self, type, name, host): """ Publish over the MQ a message to inform that a plugin configuration has been updated @param type : package type (plugin) @param name : package name @param host : host """ self.log.debug("Publish configuration update notification for {0}-{1}.{2}".format(type, name, host)) self._pub.send_event('plugin.configuration', {"type" : type, "name" : name, "host" : host, "event" : "updated"}) def reload_stats(self): self.log.debug(u"=============== reload stats") req = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action( 'reload' ) resp = req.request('xplgw', msg.get(), 100) self.log.debug(u"Reply from xplgw: {0}".format(resp)) self.log.debug(u"=============== reload stats END")
def test_user_config_file(user_home, user_entry): info("Check user config file contents") import ConfigParser config = ConfigParser.ConfigParser() config.read("/etc/domogik/domogik.cfg") #check [domogik] section dmg = dict(config.items('domogik')) database = dict(config.items('database')) rest = dict(config.items('rest')) ok("Config file correctly loaded") info("Parse [domogik] section") import domogik #Check ix xpl port is not used _check_port_availability("0.0.0.0", 3865, udp=True) ok("xPL hub IP/port is not bound by anything else") parent_conn, child_conn = Pipe() p = Process(target=_test_user_can_write, args=( child_conn, dmg['log_dir_path'], user_entry, )) p.start() p.join() assert parent_conn.recv( ), "The directory %s for log does not exist or does not have right permissions" % dmg[ 'log_dir_path'] assert dmg['log_level'] in [ 'debug', 'info', 'warning', 'error', 'critical' ], "The log_level parameter does not have a good value. Must \ be one of debug,info,warning,error,critical" ### obsolete #if not os.path.isdir(dmg['src_prefix'] + '/share/domogik'): # try: # f = os.listdir("%s/share/domogik" % dmg['src_prefix']) # f.close() # except OSError: # fail("Can't access %s/share/domogik. Check %s is available for domogik user (if you are in development mode, be sure the directory which contains the sources is available for domogik user)." % (dmg['src_prefix'],dmg['src_prefix'])) # exit() ok("[domogik] section seems good") # check [database] section info("Parse [database] section") assert database[ 'type'] == 'mysql', "Only mysql database type is supported at the moment" uid = user_entry.pw_uid os.setreuid(0, uid) old_home = os.environ['HOME'] os.environ['HOME'] = user_home from domogik.common.database import DbHelper d = DbHelper() os.setreuid(0, 0) os.environ['HOME'] = old_home assert d.get_engine( ) != None, "Engine is not set, it seems something went wrong during connection to the database" ok("[database] section seems good") # Check [rest] section info("Parse [rest] section") for ipadd in get_ip_for_interfaces(rest['interfaces'].split(",")): _check_port_availability(ipadd, rest['port']) ok("Rest server IP/port is not bound by anything else")
class XplManager(XplPlugin, MQAsyncSub): """ Statistics manager """ def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw', log_prefix="") MQAsyncSub.__init__(\ self, self.zmq, 'xplgw', \ ['client.conversion', 'client.list']) self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') # some initial data sets self.client_xpl_map = {} self.client_conversion_map = {} self._db_sensors = {} self._db_xplstats = {} # queue to store the message that needs to be ahndled for sensor checking self._sensor_queue = Queue.Queue() # all command handling params # _lock => to be sure to be thread safe # _dict => uuid to xplstat translationg # _pkt => received messages to check self._cmd_lock_d = threading.Lock() self._cmd_dict = {} self._cmd_lock_p = threading.Lock() self._cmd_pkt = {} # load some initial data from manager and db self._load_client_to_xpl_target() self._load_conversions() # create a general listener self._create_xpl_trigger() # start handling the xplmessages self._s_thread = self._SensorThread(\ self.log, self._sensor_queue, \ self.client_conversion_map, self.pub) self._s_thread.start() # start handling the command reponses in a thread self._c_thread = self._CommandThread(\ self.log, self._db, self._cmd_lock_d, \ self._cmd_lock_p, self._cmd_dict, self._cmd_pkt, self.pub) self._c_thread.start() # start the sensorthread self.ready() def on_mdp_request(self, msg): """ Method called when an mq request comes in XplPlugin also needs this info, so we need to do a passthrough """ try: XplPlugin.on_mdp_request(self, msg) if msg.get_action() == "test": pass if msg.get_action() == "cmd.send": self._send_xpl_command(msg) except Exception as exp: self.log.error(traceback.format_exc()) def on_message(self, msgid, content): """ Method called on a subscribed message """ try: if msgid == 'client.conversion': self._parse_conversions(content) elif msgid == 'client.list': self._parse_xpl_target(content) except Exception as exp: self.log.error(traceback.format_exc()) def _load_client_to_xpl_target(self): """ Request the client conversion info This is an mq req to manager """ cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('client.list.get') response = cli.request('manager', msg.get(), timeout=10) if response: self._parse_xpl_target(response.get_data()) else: self.log.error(\ u"Updating client list failed, no response from manager") def _parse_xpl_target(self, data): """ Translate the mq data info a dict for the xpl targets """ tmp = {} for cli in data: tmp[cli] = data[cli]['xpl_source'] self.client_xpl_map = tmp def _load_conversions(self): """ Request the client conversion info This is an mq req to manager """ cli = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('client.conversion.get') response = cli.request('manager', msg.get(), timeout=10) if response: self._parse_conversions(response.get_data()) else: self.log.error(\ u"Updating conversion list failed, no response from manager") def _parse_conversions(self, data): """ Translate the mq data into a dict """ tmp = {} for cli in data: tmp[cli] = data[cli] self.client_conversion_map = tmp def _send_xpl_command(self, data): """ Reply to config.get MQ req @param data : MQ req message Needed info in data: - cmdid => command id to send - cmdparams => key/value pair of all params needed for this command """ with self._db.session_scope(): self.log.info(u"Received new cmd request: {0}".format(data)) failed = False request = data.get_data() if 'cmdid' not in request: failed = "cmdid not in message data" if 'cmdparams' not in request: failed = "cmdparams not in message data" if not failed: # get the command cmd = self._db.get_command(request['cmdid']) if cmd is not None: if cmd.xpl_command is not None: xplcmd = cmd.xpl_command xplstat = self._db.get_xpl_stat(xplcmd.stat_id) if xplstat is not None: # get the device from the db dev = self._db.get_device(int(cmd.device_id)) msg = XplMessage() if not dev['client_id'] in self.client_xpl_map.keys(): self._load_client_to_xpl_target() if not dev['client_id'] in self.client_xpl_map.keys(): failed = "Can not fincd xpl source for {0} client_id".format(dev['client_id']) else: msg.set_target(self.client_xpl_map[dev['client_id']]) msg.set_source(self.myxpl.get_source()) msg.set_type("xpl-cmnd") msg.set_schema(xplcmd.schema) # static paramsw for par in xplcmd.params: msg.add_data({par.key : par.value}) # dynamic params for par in cmd.params: if par.key in request['cmdparams']: value = request['cmdparams'][par.key] # chieck if we need a conversion if par.conversion is not None and par.conversion != '': if dev['client_id'] in self.client_conversion_map: if par.conversion in self.client_conversion_map[dev['client_id']]: exec(self.client_conversion_map[dev['client_id']][par.conversion]) value = locals()[par.conversion](value) msg.add_data({par.key : value}) else: failed = "Parameter ({0}) for device command msg is not provided in the mq message".format(par.key) if not failed: # send out the msg self.log.debug(u"Sending xplmessage: {0}".format(msg)) self.myxpl.send(msg) xplstat = self._db.detach(xplstat) # generate an uuid for the matching answer published messages if xplstat != None: resp_uuid = uuid4() self._cmd_lock_d.acquire() self._cmd_dict[str(resp_uuid)] = xplstat self._cmd_lock_d.release() else: resp_uuid = None # send the response reply_msg = MQMessage() reply_msg.set_action('cmd.send.result') reply_msg.add_data('uuid', str(resp_uuid)) reply_msg.add_data('status', True) reply_msg.add_data('reason', None) self.log.debug(u"mq reply (success) : {0}".format(reply_msg.get())) self.reply(reply_msg.get()) if failed: self.log.error(failed) reply_msg = MQMessage() reply_msg.set_action('cmd.send.result') reply_msg.add_data('uuid', None) reply_msg.add_data('status', False) reply_msg.add_data('reason', failed) self.log.debug(u"mq reply (failed) : {0}".format(reply_msg.get())) self.reply(reply_msg.get()) def _create_xpl_trigger(self): """ Create a listener to catch all xpl-stats and xpl-trig messages """ Listener(self._xpl_callback, self.myxpl, {'xpltype': 'xpl-trig'}) Listener(self._xpl_callback, self.myxpl, {'xpltype': 'xpl-stat'}) def _xpl_callback(self, pkt): """ The callback for the xpl messages push them into the needed queues """ item = {} item["msg"] = pkt item["clientId"] = next((cli for cli, xpl in self.client_xpl_map.items() if xpl == pkt.source), None) self._sensor_queue.put(item) self.log.debug(u"Adding new message to the sensorQueue, current length = {0}".format(self._sensor_queue.qsize())) #self.log.debug(u"Adding new message to the sensorQueue, current length = {0}, message = {1}".format(self._sensor_queue.qsize(), pkt)) self._cmd_lock_p.acquire() # only do this when we have outstanding commands if len(self._cmd_dict) > 0: self._cmd_pkt[time.time()] = pkt self.log.debug(u"Adding new message to the cmdQueue, current length = {0}".format(len(self._cmd_dict))) #self.log.debug(u"Adding new message to the cmdQueue, current length = {0}, message = {1}".format(len(self._cmd_dict), pkt)) self._cmd_lock_p.release() class _CommandThread(threading.Thread): """ commandthread class Class responsible for handling one xpl command """ def __init__(self, log, db, lock_d, lock_p, dic, pkt, pub): threading.Thread.__init__(self) self._db = DbHelper() self._log = log self._lock_d = lock_d self._lock_p = lock_p self._dict = dic self._pkt = pkt self._pub = pub def run(self): while True: # remove old pkts self._lock_p.acquire() for pkt in self._pkt.keys(): if pkt < time.time() - CMDTIMEOUT: self._log.warning(u"Delete packet too old (timeout reached) : {0}".format(pkt)) del(self._pkt[pkt]) self._lock_p.release() # now try to match if we have enough data if len(self._dict) > 0 and len(self._pkt) > 0: todel_pkt = [] todel_dict = [] for uuid, search in self._dict.items(): for tim, pkt in self._pkt.items(): if search.schema == pkt.schema: found = True for par in search.params: if par.key not in pkt.data: if par.value != pkt.data[par.key]: found = False elif par.multiple is not None and len(par.multiple) == 1: if pkt.data[par.key] not in par.value.split(par.multiple): found = False if found: self._log.info(u"Found response message to command with uuid: {0}".format(uuid)) # publish the result self._pub.send_event('command.result', \ {"uuid" : uuid}) todel_pkt.append(tim) todel_dict.append(uuid) # now go and delete the unneeded data self._lock_p.acquire() for tim in todel_pkt: if tim in self._pkt: del(self._pkt[tim]) #self._log.debug(u"Deleting message from the cmdQueue, current length = {0}".format(len(self._pkt))) # TODO : remove or comment the 2 following lines #self._log.debug(u"Data to delete : {0}".format(todel_dict)) #self._log.debug(u"Content before deletion : {0}".format(self._dict)) self._lock_p.release() self._lock_d.acquire() for tim in todel_dict: if tim in self._dict: del(self._dict[tim]) self._lock_d.release() todel_pkt = [] todel_dict = [] else: # nothing todo, sleep a second time.sleep(1) class _SensorThread(threading.Thread): """ SensorThread class Class that will handle the sensor storage in a seperated thread This will get messages from the SensorQueue """ def __init__(self, log, queue, conv, pub): threading.Thread.__init__(self) self._db = DbHelper() self._log = log self._queue = queue self._conv = conv self._pub = pub def _find_storeparam(self, item): #print("ITEM = {0}".format(item['msg'])) found = False tostore = [] for xplstat in self._db.get_all_xpl_stat(): sensors = 0 matching = 0 statics = 0 if xplstat.schema == item["msg"].schema: #print(" XPLSTAT = {0}".format(xplstat)) # we found a possible xplstat # try to match all params and try to find a sensorid and a vlaue to store for param in xplstat.params: #print(" PARAM = {0}".format(param)) ### Caution ! # in case you, who are reading this, have to debug something like that : # 2015-08-16 22:04:26,190 domogik-xplgw INFO Storing stat for device 'Garage' (6) and sensor 'Humidity' (69): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,306 domogik-xplgw INFO Storing stat for device 'Salon' (10) and sensor 'Humidity' (76): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,420 domogik-xplgw INFO Storing stat for device 'Chambre d'Ewan' (11) and sensor 'Humidity' (80): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,533 domogik-xplgw INFO Storing stat for device 'Chambre des parents' (12) and sensor 'Humidity' (84): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,651 domogik-xplgw INFO Storing stat for device 'Chambre de Laly' (13) and sensor 'Humidity' (88): key 'current' with value '53' after conversion. # 2015-08-16 22:04:26,770 domogik-xplgw INFO Storing stat for device 'Entrée' (17) and sensor 'Humidity' (133): key 'current' with value '53' after conversion. # # which means that for a single xPL message, the value is stored in several sensors (WTF!!! ?) # It can be related to the fact that the device address key is no more corresponding between the plugin (info.json and xpl sent by python) and the way the device was create in the databse # this should not happen, but in case... well, we may try to find a fix... if param.key in item["msg"].data and param.static: statics = statics + 1 if param.multiple is not None and len(param.multiple) == 1 and item["msg"].data[param.key] in param.value.split(param.multiple): matching = matching + 1 elif item["msg"].data[param.key] == param.value: matching = matching + 1 # now we have a matching xplstat, go and find all sensors if matching == statics: #print(" MATHING !!!!!") for param in xplstat.params: if param.key in item["msg"].data and not param.static: #print(" ===> TOSTORE !!!!!!!!! : {0}".format({'param': param, 'value': item["msg"].data[param.key]})) tostore.append( {'param': param, 'value': item["msg"].data[param.key]} ) if len(tostore) > 0: found = True if found: return (found, tostore) else: return False def run(self): while True: try: item = self._queue.get() self._log.debug(u"Getting item from the sensorQueue, current length = {0}".format(self._queue.qsize())) # if clientid is none, we don't know this sender so ignore # TODO check temp disabled until external members are working #if item["clientId"] is not None: if True: with self._db.session_scope(): fdata = self._find_storeparam(item) if fdata: #// ICI !!! self._log.debug(u"Found a matching sensor, so starting the storage procedure. Sensor : {0}".format(fdata)) for data in fdata[1]: value = data['value'] storeparam = data['param'] current_date = calendar.timegm(time.gmtime()) store = True if storeparam.ignore_values: if value in eval(storeparam.ignore_values): self._log.debug(u"Value {0} is in the ignore list {0}, so not storing.".format(value, storeparam.ignore_values)) store = False if store: # get the sensor and dev sen = self._db.get_sensor(storeparam.sensor_id) dev = self._db.get_device(sen.device_id) # check if we need a conversion if sen.conversion is not None and sen.conversion != '': if dev['client_id'] in self._conv and sen.conversion in self._conv[dev['client_id']]: self._log.debug( \ u"Calling conversion {0}".format(sen.conversion)) exec(self._conv[dev['client_id']][sen.conversion]) value = locals()[sen.conversion](value) self._log.info( \ u"Storing stat for device '{0}' ({1}) and sensor '{2}' ({3}): key '{4}' with value '{5}' after conversion." \ .format(dev['name'], dev['id'], sen.name, sen.id, storeparam.key, value)) # do the store try: self._db.add_sensor_history(\ storeparam.sensor_id, \ value, \ current_date) except Exception as exp: self._log.error(u"Error when adding sensor history : {0}".format(traceback.format_exc())) else: self._log.debug(u"Don't need to store this value") # publish the result self._pub.send_event('device-stats', \ {"timestamp" : current_date, \ "device_id" : dev['id'], \ "sensor_id" : sen.id, \ "stored_value" : value}) except Queue.Empty: # nothing in the queue, sleep for 1 second time.sleep(1) except Exception as exp: self._log.error(traceback.format_exc())
ret = {} for d in devs: d = list(d) if d[0] not in ret: ret[d[0]] = {} ret[d[0]]['name'] = d[1] ret[d[0]]['keys'] = [] ret[d[0]]['keys'].append( d[2] ) else: ret[d[0]]['keys'].append( d[2] ) return ret if __name__ == "__main__": # 0- connect to the DB db = DbHelper() db.open_session() # 1- list current devices (odl ones) do = True old_devs = corellateOld(db.upgrade_list_old()) while do: print old_devs # show dev menu i = 1 for dev in old_devs: print("{0}. {1}".format(i, old_devs[dev]['name'])) i = i + 1 print("0. Exit") sel = i + 1 while sel > i:
class DBConnector(XplPlugin, MQRep): ''' Manage the connection between database and the xPL stuff Should be the *only* object along with the StatsManager to access to the database on the core side ''' def __init__(self): ''' Initialize database and xPL connection ''' XplPlugin.__init__(self, 'dbmgr') MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug("Init database_manager instance") # Check for database connexion self._db = DbHelper() nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test %s/%s" % (nb_test, DATABASE_CONNECTION_NUM_TRY) print(msg) self.log.error(msg) msg = "Waiting for %s seconds" % DATABASE_CONNECTION_WAIT print(msg) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" print(msg) self.log.error(msg) self.force_leave() return msg = "Connected to the database" print(msg) self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error("Error while starting database engine : %s" % traceback.format_exc()) self.force_leave() return Listener(self._request_config_cb, self.myxpl, {'schema': 'domogik.config', 'xpltype': 'xpl-cmnd'}) self.enable_hbeat() IOLoop.instance().start() def on_mdp_request(self, msg): if msg.get_action() == "config.get": plugin = msg._data['plugin'] hostname = msg._data['hostname'] key = msg._data['key'] if 'element' in msg._data.keys(): element = msg._data['element'] else: element = None if element: self._mdp_reply(plugin, hostname, key, self._fetch_elmt_config(plugin, element, key), element) elif not key: print 'TODO' else: self._mdp_reply(plugin, hostname, key, self._fetch_techno_config(plugin, hostname, key)) elif msg._action == "config.set": print 'TODO' def _mdp_reply(self, plugin, hostname, key, value, element=None): msg = MQMessage() msg.setaction( 'config.result' ) msg.add_data('plugin', plugin) msg.add_data('hostname', hostname) msg.add_data('key', key) msg.add_data('value', value) if element: msg.add_data('element', element) print msg.get() self.reply(msg.get()) def _request_config_cb(self, message): ''' Callback to receive a request for some config stuff @param message : the xPL message ''' #try: self._db = DbHelper(engine=self._engine) techno = message.data['plugin'] hostname = message.data['hostname'] key = message.data['key'] msg = "Request h=%s, t=%s, k=%s" % (hostname, techno, key) print(msg) self.log.debug(msg) if "value" in message.data: new_value = message.data['value'] else: new_value = None if "element" in message.data: element = message.data['element'] else: element = None msg = "Request h=%s, t=%s, k=%s (2)" % (hostname, techno, key) print(msg) self.log.debug(msg) # Set configuration if new_value: msg = "Set config h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, new_value) print msg self.log.debug(msg) self._set_config(techno, hostname, key, new_value) # Send configuration else: msg = "Request h=%s, t=%s, k=%s (send)" % (hostname, techno, key) print(msg) self.log.debug(msg) if element: msg = "Request h=%s, t=%s, k=%s (send if element)" % (hostname, techno, key) print(msg) self.log.debug(msg) self._send_config(techno, hostname, key, self._fetch_elmt_config(techno, element, key), element) else: msg = "Request h=%s, t=%s, k=%s (send else)" % (hostname, techno, key) print(msg) self.log.debug(msg) if not key: msg = "Request h=%s, t=%s, k=%s (send if not key)" % (hostname, techno, key) print(msg) self.log.debug(msg) keys = self._fetch_techno_config(techno, hostname, key).keys() values = self._fetch_techno_config(techno, hostname, key).values() self._send_config(techno, hostname, keys, values) else: msg = "Request h=%s, t=%s, k=%s (send else of if not key)" % (hostname, techno, key) print(msg) self.log.debug(msg) self._send_config(techno, hostname, key, self._fetch_techno_config(techno, hostname, key)) def _send_config(self, plugin, hostname, key, value, element = None): ''' Send a config value message for an element's config item @param plugin : the plugin of the element @param hostname : hostname @param element : the name of the element @param key : the key or list of keys of the config tuple(s) to fetch @param value : the value or list of values corresponding to the key(s) ''' msg = "Response h=%s, t=%s, k=%s, v=%s" % (hostname, plugin, key, value) print(msg) self.log.debug(msg) mess = XplMessage() mess.set_type('xpl-stat') mess.set_schema('domogik.config') mess.add_data({'plugin' : plugin}) mess.add_data({'hostname' : hostname}) if element: mess.add_data({'element' : element}) # If key/value are lists, then we add a key=value for each item if isinstance(key, list): for (_key, _val) in zip(key, value): mess.add_data({_key : _val}) else: mess.add_data({key : value}) # mess.set_conf_key('target', plugin) self.myxpl.send(mess) def _fetch_elmt_config(self, techno, element, key): ''' Fetch an element's config value in the database @param techno : the plugin of the element @param element : the name of the element @param key : the key of the config tuple to fetch ''' #TODO : use the database vals = {'x10': {'a3': {}, 'a2': {}, } } return vals[techno][element][key] def _fetch_techno_config(self, techno, hostname, key): ''' Fetch a plugin global config value in the database @param techno : the plugin of the element @param hostname : hostname @param key : the key of the config tuple to fetch ''' # This array is here for information only but is not used anymore # Values are now on the database #vals = {'x10': {'heyu-cfg-path':'/etc/heyu/x10.conf', # 'heyu-file-0': 'TTY /dev/ttyUSB0', # 'heyu-file-1': 'TTY_AUX /dev/ttyUSB0 RFXCOM', # 'heyu-file-2': 'ALIAS back_door D5 DS10A 0x677'}, # 'global': {'pid-dir-path': '/var/run/'}, # 'onewire': {'temperature_refresh_delay' : '10'}, # 'cidmodem': {'device' : '/dev/ttyUSB1', # 'nbmaxtry' : '10', # 'interval' : '15'}, # 'mirror': {'device' : '/dev/hidraw0', # 'nbmaxtry' : '10', # 'interval' : '15'}, # 'xbmc_not': {'address' : '192.168.0.20:8080', # 'delay' : '15', # 'maxdelay' : '20'}, # 'gagenda': {'email' : "*****@*****.**", # 'password' : 'XXXXXXXX', # 'calendarname' : '*****@*****.**', # 'startup-plugin':'True'}, # 'teleinfo' : {'device' : '/dev/teleinfo', # 'interval' : '30'}, # 'dawndusk' : {'startup-plugin':'True'}, # 'plcbus' : {'device':'/dev/ttyUSB0'}, # } self.log.debug("FTC 1") try: if key: self.log.debug("FTC 2") try: self.log.debug("Get plg conf for %s / %s / %s" % (techno, hostname, key)) result = self._db.get_plugin_config(techno, hostname, key) # tricky loop as workaround for a (sqlalchemy?) bug : # sometimes the given result is for another plugin/key # so while we don't get the good data, we loop # This bug happens rarely while result.id != techno or \ result.hostname != hostname or \ result.key != key: self.log.debug("Bad result : %s/%s != %s/%s" % (result.id, result.key, plugin, key)) result = self._db.get_plugin_config(techno, hostname, key) self.log.debug("Get plg conf for %s / %s / %s Result=%s" % (techno, hostname, key, result)) val = result.value self.log.debug("Get plg conf for %s / %s / %s = %s" % (techno, hostname, key, val)) if val == '': val = "None" self.log.debug("Get plg conf for %s / %s / %s = %s (2)" % (techno, hostname, key, val)) return val except AttributeError: self.log.debug("Attribute error for %s / %s / %s" % (techno, hostname, key)) return "None" else: self.log.debug("FTC 3") vals = self._db.list_plugin_config(techno, hostname) res = {} for val in vals: if val == '': res[val.key] = "None" else: res[val.key] = val.value return res except: msg = "No config found h=%s, t=%s, k=%s" % (hostname, techno, key) print(msg) self.log.warn(msg) return "None" def _set_config(self, plugin, hostname, key, value): ''' Send a config value message for an element's config item @param plugin : the plugin of the element @param hostname : hostname @param key : the key to set @param value : the value to set ''' try: self._db.set_plugin_config(technology, hostname, key, value) mess = XplMessage() mess.set_type('xpl-stat') mess.set_schema('domogik.config') mess.add_data({'plugin' : plugin}) mess.add_data({'hostname' : hostname}) mess.add_data({'key' : key}) mess.add_data({'value' : value}) self.myxpl.send(mess) except: traceback.print_exc() msg = "Error while setting h=%s, t=%s, k=%s, v=%s" % (hostname, techno, key, value) print(msg) self.log.warn(msg) return "None"
class DBConnector(XplPlugin, MQRep): ''' Manage the connection between database and the plugins Should be the *only* object along with the StatsManager to access to the database on the core side ''' def __init__(self): ''' Initialize database and xPL connection ''' XplPlugin.__init__(self, 'dbmgr') # Already done in XplPlugin #MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug(u"Init database_manager instance") # Check for database connexion self._db = DbHelper() with self._db.session_scope(): nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}".format(nb_test, DATABASE_CONNECTION_NUM_TRY) self.log.error(msg) msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return msg = "Connected to the database" self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error(u"Error while starting database engine : {0}".format(traceback.format_exc())) self.force_leave() return self.ready() # Already done in ready() #IOLoop.instance().start() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ with self._db.session_scope(): # XplPlugin handles MQ Req/rep also XplPlugin.on_mdp_request(self, msg) # configuration if msg.get_action() == "config.get": self._mdp_reply_config_get(msg) elif msg.get_action() == "config.set": self._mdp_reply_config_set(msg) elif msg.get_action() == "config.delete": self._mdp_reply_config_delete(msg) # devices list elif msg.get_action() == "device.get": self._mdp_reply_devices_result(msg) def _mdp_reply_config_get(self, data): """ Reply to config.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config request not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if 'key' not in msg_data: get_all_keys = True key = "*" else: get_all_keys = False key = msg_data['key'] if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('key', key) # we let this here to display key or * depending on the case try: if get_all_keys == True: config = self._db.list_plugin_config(name, host) self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, config)) json_config = {} for elt in config: json_config[elt.key] = self.convert(elt.value) msg.add_data('data', json_config) else: value = self._fetch_techno_config(name, host, key) # temporary fix : should be done in a better way (on db side) value = self.convert(value) self.log.info(u"Get config for {0} {1} with key '{2}' : value = {3}".format(type, name, key, value)) msg.add_data('value', value) except: status = False reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format(type, name, host, key, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_set(self, data): """ Reply to config.set MQ req @param data : MQ req message """ print "#################" msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config set : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config set not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config set : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config set : missing 'host' field : {0}".format(data) if 'data' not in msg_data: status = False reason = "Config set : missing 'data' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] data = msg_data['data'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: # we add a configured key set to true to tell the UIs and plugins that there are some configuration elements self._db.set_plugin_config(name, host, "configured", True) for key in msg_data['data']: self._db.set_plugin_config(name, host, key, data[key]) self.publish_config_updated(type, name, host) except: reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format(type, name, host, traceback.format_exc()) status = False self.log.error(reason) msg.add_data('status', status) msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_delete(self, data): """ Reply to config.delete MQ req Delete all the config items for the given type, name and host @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config request not available for type={0}".format(msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: self._db.del_plugin_config(name, host) self.log.info(u"Delete config for {0} {1}".format(type, name)) self.publish_config_updated(type, name, host) except: status = False reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format(type, name, host, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _fetch_techno_config(self, name, host, key): ''' Fetch a plugin global config value in the database @param name : the plugin of the element @param host : hostname @param key : the key of the config tuple to fetch ''' try: try: result = self._db.get_plugin_config(name, host, key) # tricky loop as workaround for a (sqlalchemy?) bug : # sometimes the given result is for another plugin/key # so while we don't get the good data, we loop # This bug happens rarely while result.id != name or \ result.hostname != host or \ result.key != key: self.log.debug(u"Bad result : {0}/{1} != {2}/{3}".format(result.id, result.key, plugin, key)) result = self._db.get_plugin_config(name, host, key) val = result.value if val == '': val = "None" return val except AttributeError: # if no result is found #self.log.error(u"Attribute error : {0}".format(traceback.format_exc())) return "None" except: msg = "No config found host={0}, plugin={1}, key={2}".format(host, name, key) self.log.warn(msg) return "None" def _mdp_reply_devices_result(self, data): """ Reply to device.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('device.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Devices request : missing 'type' field : {0}".format(data) if 'name' not in msg_data: status = False reason = "Devices request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Devices request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] #if type == "plugin": # type = DMG_VENDOR_ID name = msg_data['name'] host = msg_data['host'] dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format(type, name, host)) #dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False), dev_json = dev_list print(dev_json) msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('devices', dev_json) self.reply(msg.get()) def convert(self, data): """ Do some conversions on data """ if data == "True": data = True if data == "False": data = False return data def publish_config_updated(self, type, name, host): """ Publish over the MQ a message to inform that a plugin configuration has been updated @param type : package type (plugin) @param name : package name @param host : host """ self.log.debug("Publish configuration update notification for {0}-{1}.{2}".format(type, name, host)) self._pub.send_event('plugin.configuration', {"type" : type, "name" : name, "host" : host, "event" : "updated"})
class ScenarioManager: """ Manage scenarios : create them, evaluate them, etc ... A scenario instance contains a condition, which is a boolean combination of many tests, and a list of actions Each test can be : - test on the test of any device - test on the time - action triggered by user (click on UI for ex) The test on devices are managed directly by xpl Listeners The test on time will be managed by a TimeManager The actions will be managed by an ActionManager { "condition" : { "AND" : { "OR" : { "one-uuid" : { "param_name_1" : { "token1" : "value", "token2" : "othervalue" }, "param_name_2" : { "token3" : "foo" } }, "another-uuid" : { "param_name_1" : { "token4" : "bar" } } }, "yet-another-uuid" : { "param_name_1" : { "url" : "http://google.fr", "interval" : "5" } } } }, "actions" : [ "uid-for-action" : { "param1" : "value1", "param2" : "value2" }, "uid-for-action2" : { "param3" : "value3" } ] } """ def __init__(self, log): """ Create ScenarioManager instance @param log : Logger instance """ # Keep list of conditions as name : instance self._instances = {} # an instance of the logger self.log = log # load all scenarios from the DB self._db = DbHelper() self.load_scenarios() def load_scenarios(self): """ Loads all scenarios from the db for each scenario call the create_scenario method """ try: with self._db.session_scope(): ### TEST if database is up # TODO : move in a function and use it (also used in dbmgr) nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}. The error while trying to connect to the database is : {2}".format(nb_test, DATABASE_CONNECTION_NUM_TRY, traceback.format_exc()) self.log.error(msg) msg = "Waiting for {0} seconds".format(DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return ### Do the stuff msg = "Connected to the database" self.log.info(msg) for scenario in self._db.list_scenario(): self.create_scenario(scenario.name, scenario.json, int(scenario.id), scenario.disabled, scenario.description, scenario.state) except: self.log.error(u"Error while loading the scenarios! The error is : {0}".format(traceback.format_exc())) def shutdown(self): """ Callback to shut down all parameters """ for cond in self._conditions.keys(): self.delete_scenario(cond, db_delete=False) def get_parsed_condition(self, name): """ Call cond.get_parsed_condition on the cond with name 'name' @param name : name of the Condition @return {'name':name, 'data': parsed_condition} or raise Exception """ if name not in self._conditions: raise KeyError('no key {0} in conditions table'.format(name)) else: parsed = self._conditions[name].get_parsed_condition() return {'name': name, 'data': parsed} def update_scenario(self, cid, name, json_input, dis, desc): cid = int(cid) # TODO get the current state and store it state = True if cid != 0: self.del_scenario(cid, False) return self.create_scenario(name, json_input, cid, dis, desc, state, True) def del_scenario(self, cid, doDB=True): try: cid = int(cid) if cid == 0 or cid not in self._instances.keys(): self.log.info(u"Scenario deletion : id '{0}' doesn't exist".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)} else: self._instances[cid]['instance'].destroy() del(self._instances[cid]) if doDB: with self._db.session_scope(): self._db.del_scenario(cid) self.log.info(u"Scenario {0} deleted".format(cid)) except: msg = u"Error while deleting the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc()) self.log.error(msg) return {'status': 'ERROR', 'msg': msg} def create_scenario(self, name, json_input, cid=0, dis=False, desc=None, state=False, update=False): """ Create a Scenario from the provided json. @param name : A name for the condition instance @param json_input : JSON representation of the condition The JSON will be parsed to get all the uuids, and test instances will be created. The json needs to have 2 keys: - condition => the json that will be used to create the condition instance - actions => the json that will be used for creating the actions instances @Return {'name': name} or raise exception """ ocid = cid try: self.log.info(u"Create or save scenario : name = '{1}', id = '{1}', json = '{2}'".format(name, cid, json_input)) payload = json.loads(json_input) # quick test to check if json is valid except Exception as e: self.log.error(u"Creation of a scenario failed, invallid json: {0}".format(json_input)) self.log.error(u"Error is : {0}".format(tracebeck.format_exc())) return {'status': 'ERROR', 'msg': 'invallid json'} #if 'IF' not in payload.keys(): # or 'DO' not in payload.keys(): # msg = u"the json for the scenario does not contain condition or actions for scenario {0}".format(name) # self.log.error(msg) # return {'status': 'ERROR', 'msg': msg} # db storage if int(ocid) == 0: with self._db.session_scope(): scen = self._db.add_scenario(name, json_input, dis, desc, False) cid = scen.id elif update: with self._db.session_scope(): self._db.update_scenario(cid, name, json_input, dis, desc) # create the condition itself try: scen = ScenarioInstance(self.log, cid, name, payload, dis, state, self._db) self._instances[cid] = {'name': name, 'json': payload, 'instance': scen, 'disabled': dis } self.log.debug(u"Create scenario instance {0} with payload {1}".format(name, payload)) self._instances[cid]['instance'].eval_condition() except Exception as e: if int(ocid) == 0: with self._db.session_scope(): self._db.del_scenario(cid) self.log.error(u"Creation of a scenario failed. Error is : {0}".format(traceback.format_exc())) return {'status': 'ERROR', 'msg': 'Creation of scenario failed'} # return return {'name': name, 'status': 'OK', 'cid': cid} def eval_condition(self, name): """ Evaluate a condition calling eval_condition from Condition instance @param name : The name of the condition instance @return {'name':name, 'result': evaluation result} or raise Exception """ if name not in self._conditions: raise KeyError('no key {0} in conditions table'.format(name)) else: res = self._conditions[name].eval_condition() return {'name': name, 'result': res} def list_actions(self): """ Return the list of actions @return a hash of hashes for the different actions { "module1.Action1" : { "description" : "some description of the action", "parameters" : { "param1" : { ... see get_expected_entries for details } } """ self.log.debug("ScenarioManager : list actions") res = {} actions = self.__return_list_of_classes(s_a) for name, cls in actions: if 'abstract' not in name.lower(): self.log.debug("- {0}".format(name)) inst = cls() res[name] = {"parameters": inst.get_expected_entries(), "description": inst.get_description()} return res def list_tests(self): """ Return the list of tests @return a hash of hashes for the different tests { "module1.Test1" : { "description" : "some description of the test", "parameters" : { "param1" : { ... see list_parameters doc for detail on this part } } """ self.log.debug("ScenarioManager : list tests") res = {} tests = self.__return_list_of_classes(s_t) for name, cls in tests: if 'abstract' not in name.lower(): self.log.debug("- {0}".format(name)) inst = cls(log = self.log) params = [] for p, i in inst.get_parameters().items(): for param, info in i['expected'].items(): params.append({ "name": "{0}.{1}".format(p, param), "description": info['description'], "type": info['type'], "values": info['values'], "filters": info['filters'], }) res[name] = {"parameters": params, "blockly": inst.get_blockly(), "description": inst.get_description()} return res def list_conditions(self): """ Return the list of conditions as JSON """ ret = [] for cid, inst in self._instances.items(): ret.append({'cid': cid, 'name': inst['name'], 'json': inst['json'], 'disabled': inst['disabled']}) return ret def enable_scenario(self, cid): try: if cid == '' or int(cid) not in self._instances.keys(): self.log.info(u"Scenario enable : id '{0}' doesn't exist".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)} else: if self._instances[int(cid)]['instance'].enable(): self._instances[int(cid)]['disabled'] = False with self._db.session_scope(): self._db.update_scenario(cid, disabled=False) self.log.info(u"Scenario {0} enabled".format(cid)) return {'status': 'OK', 'msg': u"Scenario {0} enabled".format(cid)} else: self.log.info(u"Scenario {0} already enabled".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} already enabled".format(cid)} except: msg = u"Error while enabling the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc()) self.log.error(msg) return {'status': 'ERROR', 'msg': msg} def disable_scenario(self, cid): try: if cid == '' or int(cid) not in self._instances.keys(): self.log.info(u"Scenario disable : id '{0}' doesn't exist".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)} else: if self._instances[int(cid)]['instance'].disable(): self._instances[int(cid)]['disabled'] = True with self._db.session_scope(): self._db.update_scenario(cid, disabled=True) self.log.info(u"Scenario {0} disabled".format(cid)) return {'status': 'OK', 'msg': u"Scenario {0} disabled".format(cid)} else: self.log.info(u"Scenario {0} already disabled".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} already disabled".format(cid)} except: msg = u"Error while disabling the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc()) self.log.error(msg) return {'status': 'ERROR', 'msg': msg} def test_scenario(self, cid): try: if cid == '' or int(cid) not in self._instances.keys(): self.log.info(u"Scenario test : id '{0}' doesn't exist".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)} else: self._instances[int(cid)]['instance'].test_actions() self.log.info(u"Scenario {0} actions called".format(cid)) return {'status': 'OK', 'msg': u"Scenario {0} actions called".format(cid)} except: msg = u"Error while calling actions for scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc()) self.log.error(msg) return {'status': 'ERROR', 'msg': msg} def __return_list_of_classes(self, package): """ Return the list of module/classes in a package @param package : a reference to the package that need to be explored @return a list of tuple ('modulename.Classname', <instance of class>) """ self.log.debug("Get list of classes for package : {0}".format(package)) res = [] mods = pkgutil.iter_modules(package.__path__) for module in mods: self.log.debug("- {0}".format(module)) imported_mod = importlib.import_module('.' + module[1], package.__name__) #get the list of classes in the module classes = [m for m in inspect.getmembers(imported_mod) if inspect.isclass(m[1])] # Filter in order to keep only the classes that belong to domogik package and are not abstract res.extend([(module[1] + "." + c[0], c[1]) for c in filter( lambda x: x[1].__module__.startswith("domogik.scenario.") and not x[0].startswith("Abstract"), classes)]) return res
class DBConnector(XplPlugin, MQRep): ''' Manage the connection between database and the plugins Should be the *only* object along with the StatsManager to access to the database on the core side ''' def __init__(self): ''' Initialize database and xPL connection ''' XplPlugin.__init__(self, 'dbmgr') # Already done in XplPlugin #MQRep.__init__(self, zmq.Context(), 'dbmgr') self.log.debug(u"Init database_manager instance") # Check for database connexion self._db = DbHelper() with self._db.session_scope(): nb_test = 0 db_ok = False while not db_ok and nb_test < DATABASE_CONNECTION_NUM_TRY: nb_test += 1 try: self._db.list_user_accounts() db_ok = True except: msg = "The database is not responding. Check your configuration of if the database is up. Test {0}/{1}".format( nb_test, DATABASE_CONNECTION_NUM_TRY) self.log.error(msg) msg = "Waiting for {0} seconds".format( DATABASE_CONNECTION_WAIT) self.log.info(msg) time.sleep(DATABASE_CONNECTION_WAIT) if nb_test >= DATABASE_CONNECTION_NUM_TRY: msg = "Exiting dbmgr!" self.log.error(msg) self.force_leave() return msg = "Connected to the database" self.log.info(msg) try: self._engine = self._db.get_engine() except: self.log.error( u"Error while starting database engine : {0}".format( traceback.format_exc())) self.force_leave() return self.ready() # Already done in ready() #IOLoop.instance().start() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ with self._db.session_scope(): # XplPlugin handles MQ Req/rep also XplPlugin.on_mdp_request(self, msg) # configuration if msg.get_action() == "config.get": self._mdp_reply_config_get(msg) elif msg.get_action() == "config.set": self._mdp_reply_config_set(msg) elif msg.get_action() == "config.delete": self._mdp_reply_config_delete(msg) # devices list elif msg.get_action() == "device.get": self._mdp_reply_devices_result(msg) def _mdp_reply_config_get(self, data): """ Reply to config.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config request not available for type={0}".format( msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if 'key' not in msg_data: get_all_keys = True key = "*" else: get_all_keys = False key = msg_data['key'] if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data( 'key', key ) # we let this here to display key or * depending on the case try: if get_all_keys == True: config = self._db.list_plugin_config(name, host) self.log.info( u"Get config for {0} {1} with key '{2}' : value = {3}". format(type, name, key, config)) json_config = {} for elt in config: json_config[elt.key] = self.convert(elt.value) msg.add_data('data', json_config) else: value = self._fetch_techno_config(name, host, key) # temporary fix : should be done in a better way (on db side) value = self.convert(value) self.log.info( u"Get config for {0} {1} with key '{2}' : value = {3}". format(type, name, key, value)) msg.add_data('value', value) except: status = False reason = "Error while getting configuration for '{0} {1} on {2}, key {3}' : {4}".format( type, name, host, key, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_set(self, data): """ Reply to config.set MQ req @param data : MQ req message """ print "#################" msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config set : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config set not available for type={0}".format( msg_data['type']) if 'name' not in msg_data: status = False reason = "Config set : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config set : missing 'host' field : {0}".format(data) if 'data' not in msg_data: status = False reason = "Config set : missing 'data' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] data = msg_data['data'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: # we add a configured key set to true to tell the UIs and plugins that there are some configuration elements self._db.set_plugin_config(name, host, "configured", True) for key in msg_data['data']: self._db.set_plugin_config(name, host, key, data[key]) self.publish_config_updated(type, name, host) except: reason = "Error while setting configuration for '{0} {1} on {2}' : {3}".format( type, name, host, traceback.format_exc()) status = False self.log.error(reason) msg.add_data('status', status) msg.add_data('reason', reason) self.log.debug(msg.get()) self.reply(msg.get()) def _mdp_reply_config_delete(self, data): """ Reply to config.delete MQ req Delete all the config items for the given type, name and host @param data : MQ req message """ msg = MQMessage() msg.set_action('config.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Config request : missing 'type' field : {0}".format(data) if msg_data['type'] != "plugin": status = False reason = "Config request not available for type={0}".format( msg_data['type']) if 'name' not in msg_data: status = False reason = "Config request : missing 'name' field : {0}".format(data) if 'host' not in msg_data: status = False reason = "Config request : missing 'host' field : {0}".format(data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] name = msg_data['name'] host = msg_data['host'] msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) try: self._db.del_plugin_config(name, host) self.log.info(u"Delete config for {0} {1}".format(type, name)) self.publish_config_updated(type, name, host) except: status = False reason = "Error while deleting configuration for '{0} {1} on {2} : {3}".format( type, name, host, traceback.format_exc()) self.log.error(reason) msg.add_data('reason', reason) msg.add_data('status', status) self.log.debug(msg.get()) self.reply(msg.get()) def _fetch_techno_config(self, name, host, key): ''' Fetch a plugin global config value in the database @param name : the plugin of the element @param host : hostname @param key : the key of the config tuple to fetch ''' try: try: result = self._db.get_plugin_config(name, host, key) # tricky loop as workaround for a (sqlalchemy?) bug : # sometimes the given result is for another plugin/key # so while we don't get the good data, we loop # This bug happens rarely while result.id != name or \ result.hostname != host or \ result.key != key: self.log.debug(u"Bad result : {0}/{1} != {2}/{3}".format( result.id, result.key, plugin, key)) result = self._db.get_plugin_config(name, host, key) val = result.value if val == '': val = "None" return val except AttributeError: # if no result is found #self.log.error(u"Attribute error : {0}".format(traceback.format_exc())) return "None" except: msg = "No config found host={0}, plugin={1}, key={2}".format( host, name, key) self.log.warn(msg) return "None" def _mdp_reply_devices_result(self, data): """ Reply to device.get MQ req @param data : MQ req message """ msg = MQMessage() msg.set_action('device.result') status = True msg_data = data.get_data() if 'type' not in msg_data: status = False reason = "Devices request : missing 'type' field : {0}".format( data) if 'name' not in msg_data: status = False reason = "Devices request : missing 'name' field : {0}".format( data) if 'host' not in msg_data: status = False reason = "Devices request : missing 'host' field : {0}".format( data) if status == False: self.log.error(reason) else: reason = "" type = msg_data['type'] #if type == "plugin": # type = DMG_VENDOR_ID name = msg_data['name'] host = msg_data['host'] dev_list = self._db.list_devices_by_plugin("{0}-{1}.{2}".format( type, name, host)) #dev_json = json.dumps(dev_list, cls=domogik_encoder(), check_circular=False), dev_json = dev_list print(dev_json) msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('type', type) msg.add_data('name', name) msg.add_data('host', host) msg.add_data('devices', dev_json) self.reply(msg.get()) def convert(self, data): """ Do some conversions on data """ if data == "True": data = True if data == "False": data = False return data def publish_config_updated(self, type, name, host): """ Publish over the MQ a message to inform that a plugin configuration has been updated @param type : package type (plugin) @param name : package name @param host : host """ self.log.debug( "Publish configuration update notification for {0}-{1}.{2}".format( type, name, host)) self._pub.send_event('plugin.configuration', { "type": type, "name": name, "host": host, "event": "updated" })
class ScenarioManager: """ Manage scenarios : create them, evaluate them, etc ... A scenario instance contains a condition, which is a boolean combination of many tests, and a list of actions Each test can be : - test on the test of any device - test on the time - action triggered by user (click on UI for ex) The test on devices are managed directly by xpl Listeners The test on time will be managed by a TimeManager The actions will be managed by an ActionManager { "condition" : { "AND" : { "OR" : { "one-uuid" : { "param_name_1" : { "token1" : "value", "token2" : "othervalue" }, "param_name_2" : { "token3" : "foo" } }, "another-uuid" : { "param_name_1" : { "token4" : "bar" } } }, "yet-another-uuid" : { "param_name_1" : { "url" : "http://google.fr", "interval" : "5" } } } }, "actions" : [ "uid-for-action" : { "param1" : "value1", "param2" : "value2" }, "uid-for-action2" : { "param3" : "value3" } ] } """ def __init__(self, log): """ Create ScenarioManager instance @param log : Logger instance """ # Keep list of conditions as name : instance self._instances = {} # an instance of the logger self.log = log # load all scenarios from the DB self._db = DbHelper() self.load_scenarios() def load_scenarios(self): """ Loads all scenarios from the db for each scenario call the create_scenario method """ with self._db.session_scope(): for scenario in self._db.list_scenario(): self.create_scenario(scenario.name, scenario.json, int(scenario.id), scenario.disabled, scenario.description) def shutdown(self): """ Callback to shut down all parameters """ for cond in self._conditions.keys(): self.delete_scenario(cond, db_delete=False) def get_parsed_condition(self, name): """ Call cond.get_parsed_condition on the cond with name 'name' @param name : name of the Condition @return {'name':name, 'data': parsed_condition} or raise Exception """ if name not in self._conditions: raise KeyError('no key %s in conditions table' % name) else: parsed = self._conditions[name].get_parsed_condition() return {'name': name, 'data': parsed} def update_scenario(self, cid, name, json_input, dis, desc): if int(cid) != 0: self.del_scenario(cid, False) return self.create_scenario(name, json_input, cid, dis, desc, True) def del_scenario(self, cid, doDB=True): try: if cid == '' or int(cid) not in self._instances.keys(): self.log.info(u"Scenario deletion : id '{0}' doesn't exist".format(cid)) return {'status': 'ERROR', 'msg': u"Scenario {0} doesn't exist".format(cid)} else: self._instances[int(cid)]['instance'].destroy() del(self._instances[int(cid)]) if doDB: with self._db.session_scope(): self._db.del_scenario(cid) self.log.info(u"Scenario {0} deleted".format(cid)) except: msg = u"Error while deleting the scenario id='{0}'. Error is : {1}".format(cid, traceback.format_exc()) self.log.error(msg) return {'status': 'ERROR', 'msg': msg} def create_scenario(self, name, json_input, cid=0, dis=False, desc=None, update=False): """ Create a Scenario from the provided json. @param name : A name for the condition instance @param json_input : JSON representation of the condition The JSON will be parsed to get all the uuids, and test instances will be created. The json needs to have 2 keys: - condition => the json that will be used to create the condition instance - actions => the json that will be used for creating the actions instances @Return {'name': name} or raise exception """ ocid = cid try: self.log.info(u"Create or save scenario : name = '{1}', id = '{1}', json = '{2}'".format(name, cid, json_input)) payload = json.loads(json_input) # quick test to check if json is valid except Exception as e: self.log.error(u"Creation of a scenario failed, invallid json: {0}".format(json_input)) self.log.debug(e) return {'status': 'NOK', 'msg': 'invallid json'} if 'IF' not in payload.keys() \ or 'DO' not in payload.keys(): msg = u"the json for the scenario does not contain condition or actions for scenario {0}".format(name) self.log.error(msg) return {'status': 'NOK', 'msg': msg} # db storage if int(ocid) == 0: with self._db.session_scope(): scen = self._db.add_scenario(name, json_input, dis, desc) cid = scen.id elif update: with self._db.session_scope(): self._db.update_scenario(cid, name, json_input, dis, desc) # create the condition itself try: scen = ScenarioInstance(self.log, cid, name, payload, dis) self._instances[cid] = {'name': name, 'json': payload, 'instance': scen } self.log.debug(u"Create scenario instance {0} with payload {1}".format(name, payload['IF'])) self._instances[cid]['instance'].eval_condition() except Exception as e: if int(ocid) == 0: with self._db.session_scope(): self._db.del_scenario(cid) self.log.error(u"Creation of a scenario failed") self.log.debug(e) return {'status': 'NOK', 'msg': 'Creation of scenario failed'} # return return {'name': name, 'cid': cid} def eval_condition(self, name): """ Evaluate a condition calling eval_condition from Condition instance @param name : The name of the condition instance @return {'name':name, 'result': evaluation result} or raise Exception """ if name not in self._conditions: raise KeyError('no key %s in conditions table' % name) else: res = self._conditions[name].eval_condition() return {'name': name, 'result': res} def trigger_actions(self, name): """ Trigger that will be called when a condition evaluates to True """ if name not in self._conditions_actions \ or name not in self._conditions: raise KeyError('no key %s in one of the _conditions tables table' % name) else: for action in self._conditions_actions[name]: self._actions_mapping[action].do_action( \ self._conditions[name], \ self._conditions[name].get_mapping() \ ) def list_actions(self): """ Return the list of actions @return a hash of hashes for the different actions { "module1.Action1" : { "description" : "some description of the action", "parameters" : { "param1" : { ... see get_expected_entries for details } } """ self.log.debug("ScenarioManager : list actions") res = {} actions = self.__return_list_of_classes(s_a) for name, cls in actions: self.log.debug("- {0}".format(name)) inst = cls() res[name] = {"parameters": inst.get_expected_entries(), "description": inst.get_description()} return res def list_tests(self): """ Return the list of tests @return a hash of hashes for the different tests { "module1.Test1" : { "description" : "some description of the test", "parameters" : { "param1" : { ... see list_parameters doc for detail on this part } } """ self.log.debug("ScenarioManager : list tests") res = {} tests = self.__return_list_of_classes(s_t) for name, cls in tests: self.log.debug("- {0}".format(name)) inst = cls(log = self.log) params = [] for p, i in inst.get_parameters().iteritems(): for param, info in i['expected'].iteritems(): params.append({ "name": "{0}.{1}".format(p, param), "description": info['description'], "type": info['type'], "values": info['values'], "filters": info['filters'], }) res[name] = {"parameters": params, "description": inst.get_description()} return res #for name, cls in tests: # self.log.debug("- {0}".format(name)) # inst = cls(log = self.log) # res[name] = [] # for p, i in inst.get_parameters().iteritems(): # for param, info in i['expected'].iteritems(): # res[name].append({ # "name": "{0}.{1}".format(p, param), # "description": info['description'], # "type": info['type'], # "values": info['values'], # "filters": info['filters'], # }) # inst.destroy() #return res def list_conditions(self): """ Return the list of conditions as JSON """ ret = [] for cid, inst in self._instances.iteritems(): ret.append({'cid': cid, 'name': inst['name'], 'json': inst['json']}) return ret def __return_list_of_classes(self, package): """ Return the list of module/classes in a package @param package : a reference to the package that need to be explored @return a list of tuple ('modulename.Classname', <instance of class>) """ self.log.debug("Get list of classes for package : {0}".format(package)) res = [] mods = pkgutil.iter_modules(package.__path__) for module in mods: self.log.debug("- {0}".format(module)) imported_mod = importlib.import_module('.' + module[1], package.__name__) #get the list of classes in the module classes = [m for m in inspect.getmembers(imported_mod) if inspect.isclass(m[1])] # Filter in order to keep only the classes that belong to domogik package and are not abstract res.extend([(module[1] + "." + c[0], c[1]) for c in filter( lambda x: x[1].__module__.startswith("domogik.scenario.") and not x[0].startswith("Abstract"), classes)]) return res
ret = {} for d in devs: d = list(d) if d[0] not in ret: ret[d[0]] = {} ret[d[0]]['name'] = d[1] ret[d[0]]['keys'] = [] ret[d[0]]['keys'].append(d[2]) else: ret[d[0]]['keys'].append(d[2]) return ret if __name__ == "__main__": # 0- connect to the DB db = DbHelper() db.open_session() # 1- list current devices (odl ones) do = True old_devs = corellateOld(db.upgrade_list_old()) while do: print old_devs # show dev menu i = 1 for dev in old_devs: print("{0}. {1}".format(i, old_devs[dev]['name'])) i = i + 1 print("0. Exit") sel = i + 1 while sel > i: