def __init__(self): self.ctx = zmq.Context() self.WSmessages = Queue() self.MQmessages = Queue() self.sub = MQAsyncSub.__init__(self, self.ctx, 'admin', []) self.pub = MQPub(self.ctx, 'admin-ws') self.subscribers = set()
def __init__(self, name, stop_cb=None, is_manager=False, parser=None, daemonize=True, log_prefix="interface_", log_on_stdout=True, test=False, source=None): ''' Create Interface instance, which defines system handlers @param source : overwrite the source value (client-device.instance) ''' Plugin.__init__(self, name, type="interface", stop_cb=stop_cb, is_manager=is_manager, parser=parser, daemonize=daemonize, log_prefix=log_prefix, log_on_stdout=log_on_stdout, test=test) self.log.info(u"Start of the interface init") # define the source (in can be used in some plugins) if source == None: self.source = "{0}-{1}.{2}".format(INTERFACE_VENDOR_ID, self.get_plugin_name(), self.get_sanitized_hostname()) # in case we overwrite the source : else: self.source = source ### MQ self._mq_name = self.source #self.zmq = zmq.Context() self.mq_pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs MQAsyncSub.__init__(self, self.zmq, self._name, ['interface.output']) ### Context # set the context # All elements that may be added in the request sent over MQ # * media (irc, audio, sms, ...) # * text (from voice recognition) # * location (the input element location : this is configured on the input element : kitchen, garden, bedroom, ...) # * identity (from face recognition) # * mood (from kinect or anything else) # * sex (from voice recognition and post processing on the voice) self.butler_context = { "media": None, "location": None, "identity": None, "mood": None, "sex": None } self.log.info(u"End of the interface init")
def WSPublishMetricsBrowser(self, data): pub = MQPub(zmq.Context(), "domoweb") # we add data on python side in case the web clients are not all at the same time! data['timestamp'] = time.time() # and we make sure to remove auth informations del data['rest_auth'] logging.info("Publish : 'metrics.browser' > '{0}'".format(data)) pub.send_event('metrics.browser', data) logging.info("Publish : 'metrics.browser' done")
class ButlerAction(AbstractAction): """ Mock a butler response to send notifications with the butler """ def __init__(self, log=None, params=None): AbstractAction.__init__(self, log) self.set_description("Make the butler say something.") ### Butler configuration elements try: cfg = Loader('butler') config = cfg.load() conf = dict(config[1]) self.butler_name = conf['name'] self.butler_sex = conf['sex'] except: self._log.error(u"ButlerACtion init : error while reading the configuration file '{0}' : {1}".format(CONFIG_FILE, traceback.format_exc())) ### MQ self._mq_name = "butler" self.zmq = zmq.Context() self.pub = MQPub(self.zmq, self._mq_name) def do_action(self): self._log.info("Make the butler say '{0}'. Media is '{1}', location is '{2}'".format(self._params['text'], self._params['media'], self._params['location'])) # publish over MQ data = {"media" : self._params['media'], "location" : self._params['location'], "sex" : self.butler_sex, "mood" : None, "reply_to" : None, "identity" : self.butler_name, "text" : self._params['text']} self.pub.send_event('interface.output', data) def get_expected_entries(self): return {'media': {'type': 'list', 'description': 'Media target', 'values' : [['*', '*'], ['Audio', 'audio']], 'default': '*'}, 'location': {'type': 'string', 'description': 'Location', 'default': ''}, 'text': {'type': 'string', 'description': 'Text', 'default': ''} }
def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw', log_prefix="core_") self.add_mq_sub('client.conversion') self.add_mq_sub('client.list') self.add_mq_sub('client.sensor') self.add_mq_sub('device.update') self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') # some initial data sets self.client_xpl_map = {} self.client_conversion_map = {} self._db_sensors = {} self._db_xplstats = {} # load devices informations self._reload_devices() self._reload_commands() self._reload_xpl_stats() # queue to store the message that needs to be ahndled for sensor checking self._sensor_queue = queue.Queue() # queue to handle the sensor storage self._sensor_store_queue = queue.Queue() # all command handling params # _lock => to be sure to be thread safe # _dict => uuid to xplstat translationg # _pkt => received messages to check self._cmd_lock_d = threading.Lock() self._cmd_dict = {} self._cmd_lock_p = threading.Lock() self._cmd_pkt = {} # load some initial data from manager and db self._load_client_to_xpl_target() self._load_conversions() # create a general listener self._create_xpl_trigger() # start handling the xplmessages self._x_thread = self._XplSensorThread(\ self.log, self._sensor_queue, \ self._sensor_store_queue) self._x_thread.start() # start handling the command reponses in a thread self._c_thread = self._XplCommandThread(\ self.log, self._db, self._cmd_lock_d, \ self._cmd_lock_p, self._cmd_dict, self._cmd_pkt, self.pub) self._c_thread.start() # start the sensor storage thread self._s_thread = self._SensorStoreThread(\ self._sensor_store_queue, self.log, \ self._get_conversion_map, self.pub) self._s_thread.start() # start the sensorthread self.ready()
def __init__(self, log=None, params=None): AbstractAction.__init__(self, log) self.set_description("Make the butler say something.") ### Butler configuration elements try: cfg = Loader('butler') config = cfg.load() conf = dict(config[1]) self.butler_name = conf['name'] self.butler_sex = conf['sex'] except: self._log.error(u"ButlerACtion init : error while reading the configuration file '{0}' : {1}".format(CONFIG_FILE, traceback.format_exc())) ### MQ self._mq_name = "butler" self.zmq = zmq.Context() self.pub = MQPub(self.zmq, self._mq_name)
def publishToMQ(self): ctx = zmq.Context() cli = MQSyncReq(ctx) pub = MQPub(ctx, 'admin') while True: message = yield self.MQmessages.get() jsons = json.loads(message) # req/rep if 'mq_request' in jsons and 'data' in jsons: msg = MQMessage() msg.set_action(str(jsons['mq_request'])) msg.set_data(jsons['data']) print("REQ : {0}".format(msg.get())) if 'dst' in jsons: print cli.request(str(jsons['dst']), msg.get(), timeout=10).get() else: print cli.request('manager', msg.get(), timeout=10).get() # pub elif 'mq_publish' in jsons and 'data' in jsons: print("Publish : {0}".format(jsons['data'])) pub.send_event(jsons['mq_publish'], jsons['data'])
def __init__(self): """ Initiate DbHelper, Logs and config """ XplPlugin.__init__(self, 'xplgw', log_prefix = "") MQAsyncSub.__init__(self, self.zmq, 'xplgw', ['client.conversion', 'client.list']) self.log.info(u"XPL manager initialisation...") self._db = DbHelper() self.pub = MQPub(zmq.Context(), 'xplgw') self.stats = None self.client_xpl_map = {} self.client_conversion_map = {} self._load_client_to_xpl_target() self._load_conversions() self.load() self.ready()
def __init__(self, name, stop_cb = None, is_manager = False, parser = None, daemonize = True, log_prefix = "interface_", log_on_stdout = True, test = False, source = None): ''' Create Interface instance, which defines system handlers @param source : overwrite the source value (client-device.instance) ''' Plugin.__init__(self, name, type = "interface", stop_cb = stop_cb, is_manager = is_manager, parser = parser, daemonize = daemonize, log_prefix = log_prefix, log_on_stdout = log_on_stdout, test = test) self.log.info(u"Start of the interface init") # define the source (in can be used in some plugins) if source == None: self.source = "{0}-{1}.{2}".format(INTERFACE_VENDOR_ID, self.get_plugin_name(), self.get_sanitized_hostname()) # in case we overwrite the source : else: self.source = source ### MQ self._mq_name = self.source #self.zmq = zmq.Context() self.mq_pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs MQAsyncSub.__init__(self, self.zmq, self._name, ['interface.output']) ### Context # set the context # All elements that may be added in the request sent over MQ # * media (irc, audio, sms, ...) # * text (from voice recognition) # * location (the input element location : this is configured on the input element : kitchen, garden, bedroom, ...) # * identity (from face recognition) # * mood (from kinect or anything else) # * sex (from voice recognition and post processing on the voice) self.butler_context = {"media" : None, "location" : None, "identity" : None, "mood" : None, "sex" : None } self.log.info(u"End of the interface init")
def __init__(self, name, stop_cb = None, is_manager = False, parser = None, daemonize = True, log_prefix = "", test = False): ''' Create XplPlugin instance, which defines system handlers @param name : The name of the current plugin @param stop_cb : Additionnal method to call when a stop request is received @param is_manager : Must be True if the child script is a Domogik Manager process You should never need to set it to True unless you develop your own manager @param parser : An instance of ArgumentParser. If you want to add extra options to the generic option parser, create your own ArgumentParser instance, use parser.add_argument and then pass your parser instance as parameter. Your options/params will then be available on self.options and self.args @param daemonize : If set to False, force the instance *not* to daemonize, even if '-f' is not passed on the command line. If set to True (default), will check if -f was added. @param log_prefix : If set, use this prefix when creating the log file in Logger() ''' BasePlugin.__init__(self, name, stop_cb, parser, daemonize, log_prefix) Watcher(self) self.log.info(u"----------------------------------") self.log.info(u"Starting plugin '{0}' (new manager instance)".format(name)) self.log.info(u"Python version is {0}".format(sys.version_info)) if self.options.test_option: self.log.info(u"The plugin is starting in TEST mode. Test option is {0}".format(self.options.test_option)) self._name = name self._test = test # flag used to avoid loading json in test mode ''' Calculate the MQ name - For a core component this is just its component name (self._name) - For a plugin this is plugin-<self._name>-self.hostname The reason is that the core components need a fixed name on the mq network, if a plugin starts up it needs to request the config on the network, and it needs to know the worker (core component) to ask the config from. Because of the above reason, every item in the core_component list can only run once ''' if self._name in CORE_COMPONENTS: self._mq_name = self._name else: self._mq_name = "plugin-{0}.{1}".format(self._name, self.get_sanitized_hostname()) # MQ publisher and REP self.zmq = zmq.Context() self._pub = MQPub(self.zmq, self._mq_name) self._set_status(STATUS_STARTING) # MQ : start the thread which sends the status each N seconds thr_send_status = threading.Thread(None, self._send_status_loop, "send_status_loop", (), {}) thr_send_status.start() ### MQ # for stop requests MQRep.__init__(self, self.zmq, self._mq_name) self.helpers = {} self._is_manager = is_manager cfg = Loader('domogik') my_conf = cfg.load() self._config_files = CONFIG_FILE self.config = dict(my_conf[1]) self.libraries_directory = self.config['libraries_path'] self.packages_directory = "{0}/{1}".format(self.config['libraries_path'], PACKAGES_DIR) self.resources_directory = "{0}/{1}".format(self.config['libraries_path'], RESOURCES_DIR) self.products_directory = "{0}/{1}_{2}/{3}".format(self.packages_directory, "plugin", self._name, PRODUCTS_DIR) # plugin config self._plugin_config = None # Get pid and write it in a file self._pid_dir_path = self.config['pid_dir_path'] self._get_pid() if len(self.get_sanitized_hostname()) > 16: self.log.error(u"You must use 16 char max hostnames ! {0} is {1} long".format(self.get_sanitized_hostname(), len(self.get_sanitized_hostname()))) self.force_leave() return # Create object which get process informations (cpu, memory, etc) # TODO : activate # TODO : use something else that xPL ????????? #self._process_info = ProcessInfo(os.getpid(), # TIME_BETWEEN_EACH_PROCESS_STATUS, # self._send_process_info, # self.log, # self.myxpl) #self._process_info.start() self.dont_run_ready = False # for all no core elements, load the json # TODO find a way to do it nicer ?? if self._name not in CORE_COMPONENTS and self._test == False: self._load_json() # init an empty devices list self.devices = [] # init an empty 'new' devices list self.new_devices = [] # check for products pictures if self._name not in CORE_COMPONENTS and self._test == False: self.check_for_pictures() # init finished self.log.info(u"End init of the global Plugin part")
class Plugin(BasePlugin, MQRep): ''' Global plugin class, manage signal handlers. This class shouldn't be used as-it but should be extended by no xPL plugin or by the class xPL plugin which will be used by the xPL plugins This class is a Singleton ''' def __init__(self, name, stop_cb = None, is_manager = False, parser = None, daemonize = True, log_prefix = "", test = False): ''' Create XplPlugin instance, which defines system handlers @param name : The name of the current plugin @param stop_cb : Additionnal method to call when a stop request is received @param is_manager : Must be True if the child script is a Domogik Manager process You should never need to set it to True unless you develop your own manager @param parser : An instance of ArgumentParser. If you want to add extra options to the generic option parser, create your own ArgumentParser instance, use parser.add_argument and then pass your parser instance as parameter. Your options/params will then be available on self.options and self.args @param daemonize : If set to False, force the instance *not* to daemonize, even if '-f' is not passed on the command line. If set to True (default), will check if -f was added. @param log_prefix : If set, use this prefix when creating the log file in Logger() ''' BasePlugin.__init__(self, name, stop_cb, parser, daemonize, log_prefix) Watcher(self) self.log.info(u"----------------------------------") self.log.info(u"Starting plugin '{0}' (new manager instance)".format(name)) self.log.info(u"Python version is {0}".format(sys.version_info)) if self.options.test_option: self.log.info(u"The plugin is starting in TEST mode. Test option is {0}".format(self.options.test_option)) self._name = name self._test = test # flag used to avoid loading json in test mode ''' Calculate the MQ name - For a core component this is just its component name (self._name) - For a plugin this is plugin-<self._name>-self.hostname The reason is that the core components need a fixed name on the mq network, if a plugin starts up it needs to request the config on the network, and it needs to know the worker (core component) to ask the config from. Because of the above reason, every item in the core_component list can only run once ''' if self._name in CORE_COMPONENTS: self._mq_name = self._name else: self._mq_name = "plugin-{0}.{1}".format(self._name, self.get_sanitized_hostname()) # MQ publisher and REP self.zmq = zmq.Context() self._pub = MQPub(self.zmq, self._mq_name) self._set_status(STATUS_STARTING) # MQ : start the thread which sends the status each N seconds thr_send_status = threading.Thread(None, self._send_status_loop, "send_status_loop", (), {}) thr_send_status.start() ### MQ # for stop requests MQRep.__init__(self, self.zmq, self._mq_name) self.helpers = {} self._is_manager = is_manager cfg = Loader('domogik') my_conf = cfg.load() self._config_files = CONFIG_FILE self.config = dict(my_conf[1]) self.libraries_directory = self.config['libraries_path'] self.packages_directory = "{0}/{1}".format(self.config['libraries_path'], PACKAGES_DIR) self.resources_directory = "{0}/{1}".format(self.config['libraries_path'], RESOURCES_DIR) self.products_directory = "{0}/{1}_{2}/{3}".format(self.packages_directory, "plugin", self._name, PRODUCTS_DIR) # plugin config self._plugin_config = None # Get pid and write it in a file self._pid_dir_path = self.config['pid_dir_path'] self._get_pid() if len(self.get_sanitized_hostname()) > 16: self.log.error(u"You must use 16 char max hostnames ! {0} is {1} long".format(self.get_sanitized_hostname(), len(self.get_sanitized_hostname()))) self.force_leave() return # Create object which get process informations (cpu, memory, etc) # TODO : activate # TODO : use something else that xPL ????????? #self._process_info = ProcessInfo(os.getpid(), # TIME_BETWEEN_EACH_PROCESS_STATUS, # self._send_process_info, # self.log, # self.myxpl) #self._process_info.start() self.dont_run_ready = False # for all no core elements, load the json # TODO find a way to do it nicer ?? if self._name not in CORE_COMPONENTS and self._test == False: self._load_json() # init an empty devices list self.devices = [] # init an empty 'new' devices list self.new_devices = [] # check for products pictures if self._name not in CORE_COMPONENTS and self._test == False: self.check_for_pictures() # init finished self.log.info(u"End init of the global Plugin part") def check_configured(self): """ For a plugin only To be call in the plugin __init__() Check in database (over queryconfig) if the key 'configured' is set to True for the plugin if not, stop the plugin and log this """ self._plugin_config = Query(self.zmq, self.log) configured = self._plugin_config.query(self._name, 'configured') if configured == '1': configured = True if configured != True: self.log.error(u"The plugin is not configured (configured = '{0}'. Stopping the plugin...".format(configured)) self.force_leave(status = STATUS_NOT_CONFIGURED) return False self.log.info(u"The plugin is configured. Continuing (hoping that the user applied the appropriate configuration ;)") return True def _load_json(self): """ Load the plugin json file """ try: self.log.info(u"Read the json file and validate id".format(self._name)) pkg_json = PackageJson(pkg_type = "plugin", name = self._name) # check if json is valid if pkg_json.validate() == False: # TODO : how to get the reason ? self.log.error(u"Invalid json file") self.force_leave(status = STATUS_INVALID) else: # if valid, store the data so that it can be used later self.log.info(u"The json file is valid") self.json_data = pkg_json.get_json() except: self.log.error(u"Error while trying to read the json file : {1}".format(self._name, traceback.format_exc())) self.force_leave(status = STATUS_INVALID) def get_config(self, key): """ Try to get the config over the MQ. If value is None, get the default value """ if self._plugin_config == None: self._plugin_config = Query(self.zmq, self.log) value = self._plugin_config.query(self._name, key) if value == None or value == 'None': self.log.info(u"Value for '{0}' is None or 'None' : trying to get the default value instead...".format(key)) value = self.get_config_default_value(key) self.log.info(u"Value for '{0}' is : {1}".format(key, value)) return self.cast_config_value(key, value) def get_config_default_value(self, key): """ Get the default value for a config key from the json file @param key : configuration key """ for idx in range(len(self.json_data['configuration'])): if self.json_data['configuration'][idx]['key'] == key: default = self.json_data['configuration'][idx]['default'] self.log.info(u"Default value required for key '{0}' = {1}".format(key, default)) return default def cast_config_value(self, key, value): """ Cast the config value as the given type in the json file @param key : configuration key @param value : configuration value to cast and return @return : the casted value """ for idx in range(len(self.json_data['configuration'])): if self.json_data['configuration'][idx]['key'] == key: type = self.json_data['configuration'][idx]['default'] self.log.info(u"Casting value for key '{0}' in type '{1}'...".format(key, type)) return self.cast(value, type) # no cast operation : return the value if value == "None": return None return value def cast(self, value, type): """ Cast a value for a type @param value : value to cast @param type : type in which you want to cast the value """ try: if type == "boolean": # just in case, the "True"/"False" are not already converted in True/False # this is (currently) done on queryconfig side if value == "True": return True elif value == "False": return False # type == choice : nothing to do if type == "date": self.log.error(u"TODO : the cast in date format is not yet developped. Please request fritz_smh to do it") if type == "datetime": self.log.error(u"TODO : the cast in date format is not yet developped. Please request fritz_smh to do it") # type == email : nothing to do if type == "float": return float(value) if type == "integer": return float(value) # type == ipv4 : nothing to do # type == multiple choice : nothing to do # type == string : nothing to do if type == "time": self.log.error(u"TODO : the cast in date format is not yet developped. Please request fritz_smh to do it") # type == url : nothing to do except: # if an error occurs : return the default value and log a warning self.log.warning(u"Error while casting value '{0}' to type '{1}'. The plugin may not work!! Error : {2}".format(value, type, traceback.format_exc())) return value return value def get_device_list(self, quit_if_no_device = False): """ Request the dbmgr component over MQ to get the devices list for this client @param quit_if_no_device: if True, exit the plugin if there is no devices """ self.log.info(u"Retrieve the devices list for this client...") mq_client = MQSyncReq(self.zmq) msg = MQMessage() msg.set_action('device.get') msg.add_data('type', 'plugin') msg.add_data('name', self._name) msg.add_data('host', self.get_sanitized_hostname()) result = mq_client.request('dbmgr', msg.get(), timeout=10) if not result: self.log.error(u"Unable to retrieve the device list") self.force_leave() return [] else: device_list = result.get_data()['devices'] if device_list == []: self.log.warn(u"There is no device created for this client") if quit_if_no_device: self.log.warn(u"The developper requested to stop the client if there is no device created") self.force_leave() return [] for a_device in device_list: self.log.info(u"- id : {0} / name : {1} / device type id : {2}".format(a_device['id'], \ a_device['name'], \ a_device['device_type_id'])) # log some informations about the device # notice that even if we are not in the XplPlugin class we will display xpl related informations : # for some no xpl plugins, there will just be nothing to display. # first : the stats self.log.info(u" xpl_stats features :") for a_xpl_stat in a_device['xpl_stats']: self.log.info(u" - {0}".format(a_xpl_stat)) self.log.info(u" Static Parameters :") for a_feature in a_device['xpl_stats'][a_xpl_stat]['parameters']['static']: self.log.info(u" - {0} = {1}".format(a_feature['key'], a_feature['value'])) self.log.info(u" Dynamic Parameters :") for a_feature in a_device['xpl_stats'][a_xpl_stat]['parameters']['dynamic']: self.log.info(u" - {0}".format(a_feature['key'])) # then, the commands self.log.info(u" xpl_commands features :") for a_xpl_cmd in a_device['xpl_commands']: self.log.info(u" - {0}".format(a_xpl_cmd)) self.log.info(u" + Parameters :") for a_feature in a_device['xpl_commands'][a_xpl_cmd]['parameters']: self.log.info(u" - {0} = {1}".format(a_feature['key'], a_feature['value'])) self.devices = device_list return device_list def device_detected(self, data): """ The plugin developpers can call this function when a device is detected This function will check if a corresponding device exists and : - if so, do nothing - if not, add the device in a 'new devices' list - if the device is already in the 'new devices list', does nothing - if not : add it into the list and send a MQ message : an event for the UI to say a new device is detected @param data : data about the device Data example : { "device_type" : "...", "reference" : "...", "global" : [ { "key" : "....", "value" : "...." }, ... ], "xpl" : [ { "key" : "....", "value" : "...." }, ... ], "xpl_commands" : { "command_id" : [ { "key" : "....", "value" : "...." }, ... ], "command_id_2" : [...] }, "xpl_stats" : { "sensor_id" : [ { "key" : "....", "value" : "...." }, ... ], "sensor_id_2" : [...] } } """ self.log.debug(u"Device detected : data = {0}".format(data)) # browse all devices to find if the device exists found = False for a_device in self.devices: # TODO : set the id if the device exists! pass if found: self.log.debug(u"The device already exists : id={0}.".format(a_device['id'])) else: self.log.debug(u"The device doesn't exists in database") # generate a unique id for the device from its addresses new_device_id = self.generate_detected_device_id(data) # add the device feature in the new devices list : self.new_devices[device_type][type][feature] = data self.log.debug(u"Check if the device has already be marked as new...") found = False for a_device in self.new_devices: if a_device['id'] == new_device_id: found = True #for a_device in self.new_devices: # if a_device['device_type_id'] == device_type and \ # a_device['type'] == type and \ # a_device['feature'] == feature: # # if data == a_device['data']: # found = True if found == False: new_device = {'id' : new_device_id, 'data' : data} self.log.info(u"New device feature detected and added in the new devices list : {0}".format(new_device)) self.new_devices.append(new_device) # publish new devices update self._pub.send_event('device.new', {"type" : "plugin", "name" : self._name, "host" : self.get_sanitized_hostname(), "client_id" : "plugin-{0}.{1}".format(self._name, self.get_sanitized_hostname()), "device" : new_device}) # TODO : later (0.4.0+), publish one "new device" notification with only the new device detected else: self.log.debug(u"The device has already been detected since the plugin startup") def generate_detected_device_id(self, data): """ Generate an unique id based on the content of data """ # TODO : improve to make something more sexy ? the_id = json.dumps(data, sort_keys=True) chars_to_remove = ['"', '{', '}', ',', ' ', '=', '[', ']', ':'] the_id = the_id.translate(None, ''.join(chars_to_remove)) return the_id def OLD_device_detected(self, device_type, type, feature, data): """ The plugin developpers can call this function when a device is detected This function will check if a corresponding device exists and : - if so, do nothing - if not, add the device in a 'new devices' list - if the device is already in the 'new devices list', does nothing - if not : add it into the list and send a MQ message : an event for the UI to say a new device is detected ### TODO : implement a req/rep MQ message to allow UI to get the new devices list @param device_type : device_type of the detected device @param data : data about the device (address or any other configuration element of a device for this plugin) @param type : xpl_stats, xpl_commands @param feature : a xpl_stat or xpl_command feature """ self.log.debug(u"Device detected : device_type = {0}, data = {1}".format(device_type, data)) #self.log.debug(u"Already existing devices : {0}".format(self.devices)) # browse all devices to find if the device exists found = False for a_device in self.devices: # first, search for device type if a_device['device_type_id'] == device_type: params = a_device[type][feature]['parameters']['static'] found = True for key in data: for a_param in params: if key == a_param['key'] and data[key] != a_param['value']: found = False break if found: break if found: self.log.debug(u"The device already exists : id={0}.".format(a_device['id'])) else: self.log.debug(u"The device doesn't exists in database") # add the device feature in the new devices list : self.new_devices[device_type][type][feature] = data self.log.debug(u"Check if the device has already be marked as new...") found = False for a_device in self.new_devices: if a_device['device_type_id'] == device_type and \ a_device['type'] == type and \ a_device['feature'] == feature: if data == a_device['data']: found = True if found == False: new_device ={'device_type_id' : device_type, 'type' : type, 'feature' : feature, 'data' : data} self.log.info(u"New device feature detected and added in the new devices list : {0}".format(new_device)) self.new_devices.append(new_device) # publish new devices update self._pub.send_event('device.new', {"type" : "plugin", "name" : self._name, "host" : self.get_sanitized_hostname(), "client_id" : "plugin-{0}.{1}".format(self._name, self.get_sanitized_hostname()), "device" : new_device}) # TODO : later (0.4.0+), publish one "new device" notification with only the new device detected else: self.log.debug(u"The device has already been detected since the plugin startup") def get_parameter(self, a_device, key): """ For a device feature, return the required parameter value @param a_device: the device informations @param key: the parameter key """ try: self.log.debug(u"Get parameter '{0}'".format(key)) for a_param in a_device['parameters']: if a_param == key: value = self.cast(a_device['parameters'][a_param]['value'], a_device['parameters'][a_param]['type']) self.log.debug(u"Parameter value found: {0}".format(value)) return value self.log.warning(u"Parameter not found : return None") return None except: self.log.error(u"Error while looking for a device parameter. Return None. Error: {0}".format(traceback.format_exc())) return None def get_parameter_for_feature(self, a_device, type, feature, key): """ For a device feature, return the required parameter value @param a_device: the device informations @param type: the parameter type (xpl_stats, ...) @param feature: the parameter feature @param key: the parameter key """ try: self.log.debug(u"Get parameter '{0}' for '{1}', feature '{2}'".format(key, type, feature)) for a_param in a_device[type][feature]['parameters']['static']: if a_param['key'] == key: value = self.cast(a_param['value'], a_param['type']) self.log.debug(u"Parameter value found: {0}".format(value)) return value self.log.warning(u"Parameter not found : return None") return None except: self.log.error(u"Error while looking for a device feature parameter. Return None. Error: {0}".format(traceback.format_exc())) return None def check_for_pictures(self): """ if some products are defined, check if the corresponding pictures are present in the products/ folder """ self.log.info(u"Check if there are pictures for the defined products") ok = True ok_product = None if self.json_data.has_key('products'): for product in self.json_data['products']: ok_product = False for ext in PRODUCTS_PICTURES_EXTENSIONS: file = "{0}.{1}".format(product['id'], ext) if os.path.isfile("{0}/{1}".format(self.get_products_directory(), file)): ok_product = True break if ok_product: self.log.debug(u"- OK : {0} ({1})".format(product['name'], file)) else: ok = False self.log.warning(u"- Missing : {0} ({1}.{2})".format(product['name'], product['id'], PRODUCTS_PICTURES_EXTENSIONS)) if ok == False: self.log.warning(u"Some pictures are missing!") else: if ok_product == None: self.log.info(u"There is no products defined for this plugin") def ready(self, ioloopstart=1): """ to call at the end of the __init__ of classes that inherits of this one In the XplPLugin class, this function will be completed to also activate the xpl hbeat """ if self.dont_run_ready == True: return ### send plugin status : STATUS_ALIVE # TODO : why the dbmgr has no self._name defined ??????? # temporary set as unknown to avoir blocking bugs if not hasattr(self, '_name'): self._name = "unknown" self._set_status(STATUS_ALIVE) ### Instantiate the MQ # nothing can be launched after this line (blocking call!!!!) self.log.info(u"Start IOLoop for MQ : nothing else can be executed in the __init__ after this! Make sure that the self.ready() call is the last line of your init!!!!") if ioloopstart == 1: IOLoop.instance().start() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ self.log.debug(u"MQ Request received : {0}" . format(str(msg))) ### stop the plugin if msg.get_action() == "plugin.stop.do": self.log.info(u"Plugin stop request : {0}".format(msg)) self._mdp_reply_plugin_stop(msg) elif msg.get_action() == "helper.list.get": self.log.info(u"Plugin helper list request : {0}".format(msg)) self._mdp_reply_helper_list(msg) elif msg.get_action() == "helper.help.get": self.log.info(u"Plugin helper help request : {0}".format(msg)) self._mdp_reply_helper_help(msg) elif msg.get_action() == "helper.do": self.log.info(u"Plugin helper action request : {0}".format(msg)) self._mdp_reply_helper_do(msg) elif msg.get_action() == "device.new.get": self.log.info(u"Plugin new devices request : {0}".format(msg)) self._mdp_reply_device_new_get(msg) def _mdp_reply_helper_do(self, msg): contens = msg.get_data() if 'command' in contens.keys(): if contens['command'] in self.helpers.keys(): if 'parameters' not in contens.keys(): contens['parameters'] = {} params = [] else: params = [] for key, value in contens['parameters'].items(): params.append( "{0}='{1}'".format(key, value) ) command = "self.{0}(".format(self.helpers[contens['command']]['call']) command += ", ".join(params) command += ")" result = eval(command) # run the command with all params msg = MQMessage() msg.set_action('helper.do.result') msg.add_data('command', contens['command']) msg.add_data('parameters', contens['parameters']) msg.add_data('result', result) self.reply(msg.get()) def _mdp_reply_helper_help(self, data): content = data.get_data() if 'command' in contens.keys(): if content['command'] in self.helpers.keys(): msg = MQMessage() msg.set_action('helper.help.result') msg.add_data('help', self.helpers[content['command']]['help']) self.reply(msg.get()) def _mdp_reply_plugin_stop(self, data): """ Stop the plugin @param data : MQ req message First, send the MQ Rep to 'ack' the request Then, change the plugin status to STATUS_STOP_REQUEST Then, quit the plugin by calling force_leave(). This should make the plugin send a STATUS_STOPPED if all is ok Notice that no check is done on the MQ req content : we need nothing in it as it is directly addressed to a plugin """ # check if the message is for us content = data.get_data() if content['name'] != self._name or content['host'] != self.get_sanitized_hostname(): return ### Send the ack over MQ Rep msg = MQMessage() msg.set_action('plugin.stop.result') status = True reason = "" msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('name', self._name) msg.add_data('host', self.get_sanitized_hostname()) self.log.info("Send reply for the stop request : {0}".format(msg)) self.reply(msg.get()) ### Change the plugin status self._set_status(STATUS_STOP_REQUEST) ### Try to stop the plugin # if it fails, the manager should try to kill the plugin self.force_leave() def _mdp_reply_helper_list(self, data): """ Return a list of supported helpers @param data : MQ req message """ ### Send the ack over MQ Rep msg = MQMessage() msg.set_action('helper.list.result') msg.add_data('actions', self.helpers.keys()) self.reply(msg.get()) def _mdp_reply_device_new_get(self, data): """ Return a list of new devices detected @param data : MQ req message """ ### Send the ack over MQ Rep msg = MQMessage() msg.set_action('device.new.result') msg.add_data('devices', self.new_devices) self.reply(msg.get()) def _set_status(self, status): """ Set the plugin status and send it """ # when ctrl-c is done, there is no more self._name at this point... # why ? because the force_leave method is called twice as show in the logs : # # ^CKeyBoardInterrupt # 2013-12-20 22:48:41,040 domogik-manager INFO Keyoard Interrupt detected, leave now. # Traceback (most recent call last): # File "./manager.py", line 1176, in <module> # main() # File "./manager.py", line 1173, in main # 2013-12-20 22:48:41,041 domogik-manager DEBUG force_leave called # 2013-12-20 22:48:41,044 domogik-manager DEBUG __del__ Single xpl plugin # 2013-12-20 22:48:41,045 domogik-manager DEBUG force_leave called if hasattr(self, '_name'): if self._name not in CORE_COMPONENTS: self._status = status self._send_status() def _send_status_loop(self): """ send the status each STATUS_HBEAT seconds """ # TODO : we could optimize by resetting the timer each time the status is sent # but as this is used only to check for dead plugins by the manager, it is not very important ;) while not self._stop.isSet(): self._send_status() self._stop.wait(STATUS_HBEAT) def _send_status(self): """ Send the plugin status over the MQ """ if hasattr(self, "_pub"): if self._name in CORE_COMPONENTS: #type = "core" return else: type = "plugin" self.log.debug("Send plugin status : {0}".format(self._status)) self._pub.send_event('plugin.status', {"type" : type, "name" : self._name, "host" : self.get_sanitized_hostname(), "event" : self._status}) def get_config_files(self): """ Return list of config files """ return self._config_files def get_products_directory(self): """ getter """ return self.products_directory def get_libraries_directory(self): """ getter """ return self.libraries_directory def get_packages_directory(self): """ getter """ return self.packages_directory def get_resources_directory(self): """ getter """ return self.resources_directory def get_data_files_directory(self): """ Return the directory where a plugin developper can store data files. If the directory doesn't exist, try to create it. After that, try to create a file inside it. If something goes wrong, generate an explicit exception. """ path = "{0}/{1}/{2}_{3}/data/".format(self.libraries_directory, PACKAGES_DIR, "plugin", self._name) if os.path.exists(path): if not os.access(path, os.W_OK & os.X_OK): raise OSError("Can't write in directory {0}".format(path)) else: try: os.mkdir(path, '0770') self.log.info(u"Create directory {0}.".format(path)) except: raise OSError("Can't create directory {0}.".format(path)) # Commented because : # a write test is done for each call of this function. For a plugin with a html server (geoloc for example), it # can be an issue as this makes a lot of write for 'nothing' on the disk. # We keep the code for now (0.4) for maybe a later use (and improved) #try: # tmp_prefix = "write_test"; # count = 0 # filename = os.path.join(path, tmp_prefix) # while(os.path.exists(filename)): # filename = "{}.{}".format(os.path.join(path, tmp_prefix),count) # count = count + 1 # f = open(filename,"w") # f.close() # os.remove(filename) #except : # raise IOError("Can't create a file in directory {0}.".format(path)) return path def register_helper(self, action, help_string, callback): if action not in self.helpers: self.helpers[action] = {'call': callback, 'help': help_string} def publish_helper(self, key, data): if hasattr(self, "_pub"): if self._name in CORE_COMPONENTS: type = "core" else: type = "plugin" self._pub.send_event('helper.publish', {"origin" : self._mq_name, "key": key, "data": data}) def _get_pid(self): """ Get current pid and write it to a file """ pid = os.getpid() pid_file = os.path.join(self._pid_dir_path, self._name + ".pid") self.log.debug(u"Write pid file for pid '{0}' in file '{1}'".format(str(pid), pid_file)) fil = open(pid_file, "w") fil.write(str(pid)) fil.close() def __del__(self): if hasattr(self, "log"): self.log.debug(u"__del__ Single plugin") self.log.debug(u"the stack is :") for elt in inspect.stack(): self.log.debug(u" {0}".format(elt)) # we guess that if no "log" is defined, the plugin has not really started, so there is no need to call force leave (and _stop, .... won't be created) self.force_leave() def force_leave(self, status = False, return_code = None): """ Leave threads & timers In the XplPLugin class, this function will be completed to also activate the xpl hbeat """ if hasattr(self, "log"): self.log.debug(u"force_leave called") #self.log.debug(u"the stack is : {0}".format(inspect.stack())) self.log.debug(u"the stack is :") for elt in inspect.stack(): self.log.debug(u" {0}".format(elt)) if return_code != None: self.set_return_code(return_code) self.log.info("Return code set to {0} when calling force_leave()".format(return_code)) # avoid ready() to be launched self.dont_run_ready = True # stop IOLoop #try: # IOLoop.instance().start() #except: # pass # send stopped status over the MQ if status: self._set_status(status) else: self._set_status(STATUS_STOPPED) # try to stop the thread try: self.get_stop().set() except AttributeError: pass if hasattr(self, "_timers"): for t in self._timers: if hasattr(self, "log"): self.log.debug(u"Try to stop timer {0}".format(t)) t.stop() if hasattr(self, "log"): self.log.debug(u"Timer stopped {0}".format(t)) if hasattr(self, "_stop_cb"): for cb in self._stop_cb: if hasattr(self, "log"): self.log.debug(u"Calling stop additionnal method : {0} ".format(cb.__name__)) cb() if hasattr(self, "_threads"): for t in self._threads: if hasattr(self, "log"): self.log.debug(u"Try to stop thread {0}".format(t)) try: t.join() except RuntimeError: pass if hasattr(self, "log"): self.log.debug(u"Thread stopped {0}".format(t)) #t._Thread__stop() #Finally, we try to delete all remaining threads for t in threading.enumerate(): if t != threading.current_thread() and t.__class__ != threading._MainThread: if hasattr(self, "log"): self.log.info(u"The thread {0} was not registered, killing it".format(t.name)) t.join() if hasattr(self, "log"): self.log.info(u"Thread {0} stopped.".format(t.name)) if threading.activeCount() > 1: if hasattr(self, "log"): self.log.warn(u"There are more than 1 thread remaining : {0}".format(threading.enumerate()))
class Publisher(MQAsyncSub): """Handles new data to be passed on to subscribers.""" def __init__(self): self.ctx = zmq.Context() self.WSmessages = Queue() self.MQmessages = Queue() self.sub = MQAsyncSub.__init__(self, self.ctx, 'admin', []) self.pub = MQPub(self.ctx, 'admin-ws') self.subscribers = set() def register(self, subscriber): """Register a new subscriber.""" self.subscribers.add(subscriber) def deregister(self, subscriber): """Stop publishing to a subscriber.""" try: self.subscribers.remove(subscriber) except: pass @gen.coroutine def on_message(self, did, msg): """Receive message from MQ sub and send to WS.""" yield self.WSmessages.put({"msgid": did, "content": msg}) @gen.coroutine def publishToWS(self): while True: message = yield self.WSmessages.get() if len(self.subscribers) > 0: #print(u"Pushing MQ message to {} WS subscribers...".format(len(self.subscribers))) yield [ subscriber.submit(message) for subscriber in self.subscribers ] @gen.coroutine def publishToMQ(self): while True: message = yield self.MQmessages.get() self.sendToMQ(message) def sendToMQ(self, message): try: ctx = zmq.Context() jsons = json.loads(message) # req/rep if 'mq_request' in jsons and 'data' in jsons: cli = MQSyncReq(ctx) msg = MQMessage() msg.set_action(str(jsons['mq_request'])) msg.set_data(jsons['data']) print(u"REQ : {0}".format(msg.get())) if 'dst' in jsons: dst = str(jsons['dst']) else: dst = 'manager' res = cli.request(dst, msg.get(), timeout=10) if res: print(res.get()) cli.shutdown() del cli # pub elif 'mq_publish' in jsons and 'data' in jsons: self.pub.send_event(jsons['mq_publish'], jsons['data']) except Exception as e: print(u"Error sending mq message: {0}".format(e))
"certfile": os.path.join(options.ssl_certificate), "keyfile": os.path.join(options.ssl_key) }) logger.info("Start https server on port '{0}'".format( options.ssl_port)) http_server_ssl.listen(options.ssl_port) else: logger.info("SSL not activated") # http logger.info("Start http server on port '{0}'".format(options.port)) http_server = tornado.httpserver.HTTPServer(application) http_server.listen(options.port) # Process info mq_pub = MQPub(zmq.Context(), "domoweb-engine") stop = threading.Event() process_info = ProcessInfo(os.getpid(), TIME_BETWEEN_EACH_PROCESS_STATUS, send_process_info, logger, stop) thr_send_process_info = threading.Thread(None, process_info.start, "send_process_info", (), {}) thr_send_process_info.start() # to allow killing the process info thread signal.signal(signal.SIGTERM, signal_handler) logger.info("Starting MQ Handler") MQHandler() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: logging.info("Manual stop requested. Exiting!")
def __init__(self, application, request, **kwargs): WebSocketHandler.__init__(self, application, request, **kwargs) self.io_loop = IOLoop.instance() self.pub = MQPub(zmq.Context(), 'admin')
class AdminWebSocket(WebSocketHandler, MQAsyncSub): #clients = set() def __init__(self, application, request, **kwargs): WebSocketHandler.__init__(self, application, request, **kwargs) self.io_loop = IOLoop.instance() self.pub = MQPub(zmq.Context(), 'admin') def open(self): self.id = uuid.uuid4() MQAsyncSub.__init__(self, zmq.Context(), 'admin', []) # Ping to make sure the agent is alive. self.io_loop.add_timeout(datetime.timedelta(seconds=random.randint(5,30)), self.send_ping) global ws_list ws_list.add({"id" : self.id, "ws" : self, "open" : True}) def on_connection_timeout(self): self.close() def send_ping(self): try: self.ping("a") self.ping_timeout = self.io_loop.add_timeout(datetime.timedelta(minutes=1), self.on_connection_timeout) except Exception as ex: pass def on_pong(self, data): if hasattr(self, "ping_timeout"): self.io_loop.remove_timeout(self.ping_timeout) # Wait 5 seconds before pinging again. self.io_loop.add_timeout(datetime.timedelta(seconds=5), self.send_ping) def on_close(self): global ws_list ws_list.delete(self.id) def on_message(self, msg, content=None): #print(ws_list.list()) """ This function is quite tricky It is called by both WebSocketHandler and MQASyncSub... """ try: ### websocket message (from the web) # there are the mesages send by the administration javascript part thanks to the ws.send(...) function if not content: jsons = json.loads(msg) # req/rep if 'mq_request' in jsons and 'data' in jsons: cli = MQSyncReq(zmq.Context()) msg = MQMessage() msg.set_action(str(jsons['mq_request'])) msg.set_data(jsons['data']) if 'dst' in jsons: cli.request(str(jsons['dst']), msg.get(), timeout=10).get() else: cli.request('manager', msg.get(), timeout=10).get() # pub elif 'mq_publish' in jsons and 'data' in jsons: print("Publish : {0}".format(jsons['data'])) self.pub.send_event(jsons['mq_publish'], jsons['data']) ### MQ message (from domogik) # these are the published messages which are received here else: try: self.write_message({"msgid": msg, "content": content}) except WebSocketClosedError: self.close() except: print("Error : {0}".format(traceback.format_exc()))
def __init__(self): ### Option parser parser = ArgumentParser() parser.add_argument("-i", action="store_true", dest="interactive", default=False, \ help="Butler interactive mode (must be used WITH -f).") Plugin.__init__(self, name='butler', parser=parser, log_prefix='core_') ### MQ # MQ publisher #self._mq_name = "interface-{0}.{1}".format(self._name, self.get_sanitized_hostname()) self._mq_name = "butler" #self.zmq = zmq.Context() self.pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs self.add_mq_sub('interface.input') # devices updates self.add_mq_sub('device.update') ### Configuration elements try: cfg = Loader('butler') config = cfg.load() conf = dict(config[1]) self.lang = conf['lang'] self.butler_name = conf['name'] self.log.debug(u"The butler configured name is '{0}'".format( self.butler_name)) self.butler_name_cleaned = clean_input(conf['name']) self.log.debug(u"The butler cleaned name is '{0}'".format( self.butler_name_cleaned)) self.butler_sex = conf['sex'] self.butler_mood = None if self.butler_sex not in SEX_ALLOWED: self.log.error( u"Exiting : the butler sex configured is not valid : '{0}'. Expecting : {1}" .format(self.butler_sex, SEX_ALLOWED)) self.force_leave() return except: self.log.error( u"Exiting : error while reading the configuration file '{0}' : {1}" .format(CONFIG_FILE, traceback.format_exc())) self.force_leave() return # user name (default is 'localuser') # this is not used for now on Domogik side self.user_name = "localuser" ### Prepare the brain # - validate packages # Start the brain :) self.brain = RiveScript(utf8=True) # set rivescript variables # Configure bot variables # all must be lower case.... self.log.info(u"Configuring name and sex : {0}, {1}".format( self.butler_name_cleaned.lower(), self.butler_sex.lower())) self.brain.set_variable(u"name", self.butler_name_cleaned.lower()) self.brain.set_variable(u"fullname", self.butler_name.lower()) self.brain.set_variable(u"sex", self.butler_sex.lower()) # set the PYTHONPATH sys.path.append(self.get_libraries_directory()) # load the brain self.brain_content = None self.learn_content = None self.not_understood_content = None self.load_all_brain() # shortcut to allow the core brain package to reload the brain for learning self.brain.reload_butler = self.reload # shortcut to allow the core brain package to do logging and access the devices in memory self.brain.log = self.log self.brain.devices = [] # will be loaded in self.reload_devices() # history self.history = [] # load all known devices self.reload_devices() self.log.info( u"*** Welcome in {0} world, your digital assistant! ***".format( self.butler_name)) # for chat more only #self.log.info(u"You may type /quit to let {0} have a break".format(self.butler_name)) ### Interactive mode if self.options.interactive: self.log.info(u"Launched in interactive mode : running the chat!") # TODO : run as a thread #self.run_chat() thr_run_chat = Thread(None, self.run_chat, "run_chat", (), {}) thr_run_chat.start() else: self.log.info(u"Not launched in interactive mode") ### TODO #self.add_stop_cb(self.shutdown) self.log.info(u"Butler initialized") self.ready()
class Publisher(MQAsyncSub): """Handles new data to be passed on to subscribers.""" def __init__(self): self.ctx = zmq.Context() self.WSmessages = Queue() self.MQmessages = Queue() self.sub = MQAsyncSub.__init__(self, self.ctx, 'admin', []) self.pub = MQPub(self.ctx, 'admin-ws') self.subscribers = set() def register(self, subscriber): """Register a new subscriber.""" self.subscribers.add(subscriber) def deregister(self, subscriber): """Stop publishing to a subscriber.""" try: self.subscribers.remove(subscriber) except: pass @gen.coroutine def on_message(self, did, msg): """Receive message from MQ sub and send to WS.""" yield self.WSmessages.put({"msgid": did, "content": msg}) @gen.coroutine def publishToWS(self): while True: message = yield self.WSmessages.get() if len(self.subscribers) > 0: #print(u"Pushing MQ message to {} WS subscribers...".format(len(self.subscribers))) yield [subscriber.submit(message) for subscriber in self.subscribers] @gen.coroutine def publishToMQ(self): while True: message = yield self.MQmessages.get() self.sendToMQ(message) def sendToMQ(self, message): try: ctx = zmq.Context() jsons = json.loads(message) # req/rep if 'mq_request' in jsons and 'data' in jsons: cli = MQSyncReq(ctx) msg = MQMessage() msg.set_action(str(jsons['mq_request'])) msg.set_data(jsons['data']) print(u"REQ : {0}".format(msg.get())) if 'dst' in jsons: dst = str(jsons['dst']) else: dst = 'manager' res = cli.request(dst, msg.get(), timeout=10) if res: print(res.get()) cli.shutdown() del cli # pub elif 'mq_publish' in jsons and 'data' in jsons: self.pub.send_event(jsons['mq_publish'], jsons['data']) except Exception as e: print(u"Error sending mq message: {0}".format(e))
class Interface(Plugin, MQAsyncSub): ''' For interfaces clients ''' def __init__(self, name, stop_cb=None, is_manager=False, parser=None, daemonize=True, log_prefix="interface_", log_on_stdout=True, test=False, source=None): ''' Create Interface instance, which defines system handlers @param source : overwrite the source value (client-device.instance) ''' Plugin.__init__(self, name, type="interface", stop_cb=stop_cb, is_manager=is_manager, parser=parser, daemonize=daemonize, log_prefix=log_prefix, log_on_stdout=log_on_stdout, test=test) self.log.info(u"Start of the interface init") # define the source (in can be used in some plugins) if source == None: self.source = "{0}-{1}.{2}".format(INTERFACE_VENDOR_ID, self.get_plugin_name(), self.get_sanitized_hostname()) # in case we overwrite the source : else: self.source = source ### MQ self._mq_name = self.source #self.zmq = zmq.Context() self.mq_pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs MQAsyncSub.__init__(self, self.zmq, self._name, ['interface.output']) ### Context # set the context # All elements that may be added in the request sent over MQ # * media (irc, audio, sms, ...) # * text (from voice recognition) # * location (the input element location : this is configured on the input element : kitchen, garden, bedroom, ...) # * identity (from face recognition) # * mood (from kinect or anything else) # * sex (from voice recognition and post processing on the voice) self.butler_context = { "media": None, "location": None, "identity": None, "mood": None, "sex": None } self.log.info(u"End of the interface init") def send_to_butler(self, message, identity=None, media=None, location=None, sex=None, mood=None): """ Send a message over MQ to the butler component """ self.log.info(u"Send a message to the butler : {0}".format(message)) request = self.butler_context request["text"] = message request["source"] = self.source if identity: request["identity"] = identity if media: request["media"] = media if location: request["location"] = location if sex: request["sex"] = sex if mood: request["mood"] = mood self.mq_pub.send_event('interface.input', request) def on_message(self, msgid, content): """ When a message is received from the MQ (pub/sub) """ if msgid == "interface.output": self.log.debug( u"interface.output received message : {0}".format(content)) self.process_response(content) return # TODO : handle context ### filter on location, media # if media not cli, don't process if content['media'] != "cli": return # location # no location, so no check ### Display message in the cli #print("{0}".format(content['text'])) print(u"{0} > {1}".format(BUTLER_CLI_NAME, content['text'])) ### Display the prompt for the next exchange print(u"{0} > ".format(BUTLER_CLI_USER)) def process_response(self, data): """ empty function that should be override to process the butler response """ pass
def __init__(self): ### Option parser parser = ArgumentParser() parser.add_argument("-i", action="store_true", dest="interactive", default=False, \ help="Butler interactive mode (must be used WITH -f).") Plugin.__init__(self, name = 'butler', parser = parser) ### MQ # MQ publisher #self._mq_name = "interface-{0}.{1}".format(self._name, self.get_sanitized_hostname()) self._mq_name = "butler" #self.zmq = zmq.Context() self.pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs MQAsyncSub.__init__(self, self.zmq, self._name, ['interface.input']) ### Configuration elements try: cfg = Loader('butler') config = cfg.load() conf = dict(config[1]) self.lang = conf['lang'] self.butler_name = conf['name'] self.butler_sex = conf['sex'] self.butler_mood = None if self.butler_sex not in SEX_ALLOWED: self.log.error(u"Exiting : the butler sex configured is not valid : '{0}'. Expecting : {1}".format(self.butler_sex, SEX_ALLOWED)) self.force_leave() return except: self.log.error(u"Exiting : error while reading the configuration file '{0}' : {1}".format(CONFIG_FILE, traceback.format_exc())) self.force_leave() return # user name (default is 'localuser') # this is not used for now on Domogik side self.user_name = "localuser" ### Prepare the brain # - validate packages # Start the brain :) self.brain = RiveScript(utf8=True) # set rivescript variables # Configure bot variables # all must be lower case.... self.log.info("Configuring name and sex : {0}, {1}".format(self.butler_name.lower(), self.butler_sex.lower())) self.brain.set_variable("name", self.butler_name.lower()) self.brain.set_variable("fullname", self.butler_name.lower()) self.brain.set_variable("sex", self.butler_sex.lower()) # set the PYTHONPATH sys.path.append(self.get_libraries_directory()) # load the brain self.brain_content = None self.learn_content = None self.not_understood_content = None self.load_all_brain() # shortcut to allow the core brain package to reload the brain for learning self.brain.reload_butler = self.reload # history self.history = [] print(u"*** Welcome in {0} world, your digital assistant! ***".format(self.butler_name)) print(u"You may type /quit to let {0} have a break".format(self.butler_name)) ### Interactive mode if self.options.interactive: self.log.info(u"Launched in interactive mode : running the chat!") # TODO : run as a thread #self.run_chat() thr_run_chat = Thread(None, self.run_chat, "run_chat", (), {}) thr_run_chat.start() else: self.log.info(u"Not launched in interactive mode") ### TODO #self.add_stop_cb(self.shutdown) self.log.info(u"Butler initialized") self.ready()
class Butler(Plugin, MQAsyncSub): """ Butler component TODO : * /quit /reload commands * interact with domogik : commands """ def __init__(self): ### Option parser parser = ArgumentParser() parser.add_argument("-i", action="store_true", dest="interactive", default=False, \ help="Butler interactive mode (must be used WITH -f).") Plugin.__init__(self, name = 'butler', parser = parser) ### MQ # MQ publisher #self._mq_name = "interface-{0}.{1}".format(self._name, self.get_sanitized_hostname()) self._mq_name = "butler" #self.zmq = zmq.Context() self.pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs MQAsyncSub.__init__(self, self.zmq, self._name, ['interface.input']) ### Configuration elements try: cfg = Loader('butler') config = cfg.load() conf = dict(config[1]) self.lang = conf['lang'] self.butler_name = conf['name'] self.butler_sex = conf['sex'] self.butler_mood = None if self.butler_sex not in SEX_ALLOWED: self.log.error(u"Exiting : the butler sex configured is not valid : '{0}'. Expecting : {1}".format(self.butler_sex, SEX_ALLOWED)) self.force_leave() return except: self.log.error(u"Exiting : error while reading the configuration file '{0}' : {1}".format(CONFIG_FILE, traceback.format_exc())) self.force_leave() return # user name (default is 'localuser') # this is not used for now on Domogik side self.user_name = "localuser" ### Prepare the brain # - validate packages # Start the brain :) self.brain = RiveScript(utf8=True) # set rivescript variables # Configure bot variables # all must be lower case.... self.log.info("Configuring name and sex : {0}, {1}".format(self.butler_name.lower(), self.butler_sex.lower())) self.brain.set_variable("name", self.butler_name.lower()) self.brain.set_variable("fullname", self.butler_name.lower()) self.brain.set_variable("sex", self.butler_sex.lower()) # set the PYTHONPATH sys.path.append(self.get_libraries_directory()) # load the brain self.brain_content = None self.learn_content = None self.not_understood_content = None self.load_all_brain() # shortcut to allow the core brain package to reload the brain for learning self.brain.reload_butler = self.reload # history self.history = [] print(u"*** Welcome in {0} world, your digital assistant! ***".format(self.butler_name)) print(u"You may type /quit to let {0} have a break".format(self.butler_name)) ### Interactive mode if self.options.interactive: self.log.info(u"Launched in interactive mode : running the chat!") # TODO : run as a thread #self.run_chat() thr_run_chat = Thread(None, self.run_chat, "run_chat", (), {}) thr_run_chat.start() else: self.log.info(u"Not launched in interactive mode") ### TODO #self.add_stop_cb(self.shutdown) self.log.info(u"Butler initialized") self.ready() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ try: ### rivescript files detail if msg.get_action() == "butler.scripts.get": self.log.info(u"Scripts request : {0}".format(msg)) self._mdp_reply_butler_scripts(msg) ### rivescript files detail elif msg.get_action() == "butler.reload.do": self.log.info(u"Reload brain request : {0}".format(msg)) self._mdp_reply_butler_reload(msg) ### history elif msg.get_action() == "butler.history.get": self.log.info(u"Get butler history : {0}".format(msg)) self._mdp_reply_butler_history(msg) ### features elif msg.get_action() == "butler.features.get": self.log.info(u"Get butler features : {0}".format(msg)) self._mdp_reply_butler_features(msg) except: self.log.error(u"Error while processing MQ message : '{0}'. Error is : {1}".format(msg, traceback.format_exc())) def _mdp_reply_butler_scripts(self, message): """ Send the raw content for the brain parts over the MQ """ # TODO : handle choice of the client in the req message # load not understood queries data self.read_not_understood_file() msg = MQMessage() msg.set_action('butler.scripts.result') msg.add_data("learn", self.learn_content) msg.add_data("not_understood", self.not_understood_content) for client_id in self.brain_content: msg.add_data(client_id, self.brain_content[client_id]) self.reply(msg.get()) def _mdp_reply_butler_reload(self, message): """ Reload the brain """ msg = MQMessage() msg.set_action('butler.reload.result') try: self.reload() msg.add_data(u"status", True) msg.add_data(u"reason", "") except: msg.add_data(u"status", False) msg.add_data(u"reason", "Error while reloading brain parts : {0}".format(traceback.format_exc())) self.reply(msg.get()) def _mdp_reply_butler_history(self, message): """ Butler history """ msg = MQMessage() msg.set_action('butler.history.result') msg.add_data("history", self.history) self.reply(msg.get()) def _mdp_reply_butler_features(self, message): """ Butler features """ msg = MQMessage() msg.set_action('butler.features.result') msg.add_data("features", self.butler_features) self.reply(msg.get()) def reload(self): self.load_all_brain() def load_all_brain(self): """ Load all the brain parts (included in domogik or in packages) and do any other related actions """ try: # load the minimal brain self.brain_content = {} # the brain raw content for display in the admin (transmitted over MQ) self.log.info(u"Load minimal brain : {0}".format(MINIMAL_BRAIN)) self.brain.load_file(MINIMAL_BRAIN) # load packages for the brain self.load_brain_parts() # sort replies self.brain.sort_replies() except: self.log.error(u"Error while loading brain : {0}".format(traceback.format_exc())) def load_brain_parts(self): """ Load the parts of the brain from /var/lib/domogik/domogik_packages/brain_* and also plugin_* because some plugins may need dedicated brain parts : - weather forecast - anything less generic than a datatype basic usage """ try: list = [] # first load the packages parts for a_file in os.listdir(self.get_packages_directory()): try: pkg_type, name = a_file.split("_") except ValueError: # not a foo_bar file : skip it continue #if a_file[0:len(BRAIN_PKG_TYPE)] == BRAIN_PKG_TYPE: if pkg_type in BRAIN_PKG_TYPES: client_id = "{0}-{1}.{2}".format(pkg_type, a_file.split("_")[1], self.get_sanitized_hostname()) self.brain_content[client_id] = {} rs_dir = os.path.join(self.get_packages_directory(), a_file, RIVESCRIPT_DIR) if os.path.isdir(rs_dir): self.log.info(u"Brain part found : {0}".format(a_file)) #self.log.debug(u"The brain part contains a rivescript folder ({0})".format(RIVESCRIPT_DIR)) lang_dir = os.path.join(rs_dir, self.lang) if os.path.isdir(lang_dir): self.log.info(u"- Language found : {0}".format(self.lang)) # add the brain part to rivescript self.brain.load_directory(lang_dir) # add the files raw data to brain content (to be sent over MQ to the admin) self.brain_content[client_id][self.lang] = {} for a_rs_file in os.listdir(lang_dir): a_rs_file_path = os.path.join(lang_dir, a_rs_file) if os.path.isfile(a_rs_file_path) and a_rs_file[-len(RIVESCRIPT_EXTENSION):] == RIVESCRIPT_EXTENSION: try: import codecs file = codecs.open(a_rs_file_path, 'r', 'utf-8') file_content = file.read() content = u"{0}".format(file_content) except: content = u"Error while reading file '{0}'. Error is : {1}".format(a_rs_file_path, traceback.format_exc()) self.log.error(content) self.brain_content[client_id][self.lang][a_rs_file] = content # and finally, load the learning file # this file is generated over the time when the domogik.butler.brain learn() function is called learn_file = LEARN_FILE if os.path.isfile(learn_file): self.log.info(u"Learn file found : {0}".format(learn_file)) # add the brain part to rivescript self.brain.load_file(learn_file) try: import codecs file = codecs.open(learn_file, 'r', 'utf-8') file_content = file.read() file_header = "// File : {0}".format(learn_file) self.learn_content = u"{0}\n\n{1}".format(file_header, file_content) except: self.learn_content = u"Error while reading file '{0}'. Error is : {1}".format(learn_file, traceback.format_exc()) self.log.error(self.learn_content) else: self.learn_content = u"" self.log.info(u"Learn file NOT found : {0}. This is not an error. You just have learn nothing to your butler ;)".format(learn_file)) # to finish, find all the tagged features # and all the tagged suggestions self.get_brain_features_and_suggestions() except: msg = "Error accessing packages directory : {0}. You should create it".format(str(traceback.format_exc())) self.log.error(msg) def get_brain_features_and_suggestions(self): """ Extract brain features and suggestions from the rivescript files : // ##feature## a feature + feature trigger - feature response /* ##suggest## ? ... @ ... */ """ self.butler_features = [] self.butler_suggestions = [] try: self.log.info(u"Extract tagged features (##feature##) and suggestions (##suggest##) from the rivescript files") for client in self.brain_content: for lang in self.brain_content[client]: for fic in self.brain_content[client][lang]: the_suggests = re.findall(SUGGEST_REGEXP, self.brain_content[client][lang][fic]) if the_suggests != []: self.butler_suggestions.extend(the_suggests) for line in self.brain_content[client][lang][fic].split("\n"): if re.search(FEATURE_TAG, line): self.butler_features.append(line.split(FEATURE_TAG)[1]) self.log.info(u"{0} feature(s) found".format(len(self.butler_features))) self.log.info(u"{0} suggestion(s) found".format(len(self.butler_suggestions))) # store in the Rivescript object the features and suggestions to be able to grab them from the core brain package self.brain.the_features = '.\n'.join(self.butler_features) self.brain.the_suggestions = self.butler_suggestions except: self.log.error(u"Error while extracting the features : {0}".format(traceback.format_exc())) def process(self, query): """ Process the input query by calling rivescript brain @param query : the text query """ try: self.log.debug(u"Before transforming query : {0}".format(query)) self.brain.raw_query = query query = clean_input(query) self.log.debug(u"After transforming query : {0}".format(query)) self.brain.query = query # process the query self.log.debug(u"Before calling Rivescript brain for processing : {0} (type={1})".format(query, type(query))) reply = self.brain.reply(self.user_name, query) self.log.debug(u"Processing finished. The reply is : {0}".format(reply)) return reply except: self.log.error(u"Error while processing query '{0}'. Error is : {1}".format(query, traceback.format_exc())) self.log.error(reply) self.log.error(type(reply)) return "Error" def shutdown(self): """ Shutdown the butler """ pass def add_to_history(self, msgid, data): self.history.append({"msgid" : msgid, "context" : data}) def on_message(self, msgid, content): """ When a message is received from the MQ (pub/sub) """ if msgid == "interface.input": self.log.info(u"Received message : {0}".format(content)) ### Get response from the brain # TODO : do it in a thread and if this last too long, do : # 3s : reply "humm..." # 10s : reply "I am checking..." # 20s : reply "It takes already 20s for processing, I cancel the request" and kill the thread #reply = self.brain.reply(self.user_name, content['text']) self.add_to_history("interface.input", content) reply = self.process(content['text']) ### Prepare response for the MQ # All elements that may be added in the request sent over MQ for interface.output # * media (irc, audio, sms, ...) # * text (from voice recognition) # * location (the input element location : this is configured on the input element : kitchen, garden, bedroom, ...) # * reply_to (from face recognition) #self.context = {"media" : "irc", # "location" : "internet", # "reply_to" : content['identity'] # } #self.context['text'] = reply # fill empty data for elt in ['identity', 'media', 'location', 'sex', 'mood', 'reply_to']: if elt not in content: content[elt] = None # publish over MQ data = {"media" : content['media'], "location" : content['location'], "sex" : self.butler_sex, "mood" : self.butler_mood, "reply_to" : content['source'], "identity" : self.butler_name, "text" : reply} self.log.info(u"Send response over MQ : {0}".format(data)) self.pub.send_event('interface.output', data) self.add_to_history("interface.output", data) def run_chat(self): """ Nestor starts to serve his master over the chat the chat is really usefull for debugging TODO : allow Nestor to connect over irc on demand for test purpose """ # just wait for 2s to have a cleaner output #time.sleep(2) # start serving for an entire life while True: msg = raw_input(u"You > ") # first, let python handle some system messages if msg == '/quit': quit() # then, let Nestor do his work!!! #reply = self.brain.reply(self.user_name, msg) reply = self.process(msg) # let Nestor answer in the chat print(u"{0} > {1}".format(self.butler_name, reply)) # let Nestor speak #tts = u"espeak -p 40 -s 140 -v mb/mb-fr1 \"{0}\" | mbrola /usr/share/mbrola/fr1/fr1 - -.au | aplay".format(reply) #tts = u"espeak -p 40 -s 140 -v mb/mb-fr1 \"{0}\" | mbrola /usr/share/mbrola/fr1 - -.au | aplay".format(reply) #subp = Popen(tts, shell=True) #pid = subp.pid #subp.communicate() def read_not_understood_file(self): """ Get the content of the non understood queries file """ if os.path.isfile(STAR_FILE): self.log.info(u"Not understood queries file found : {0}".format(STAR_FILE)) try: import codecs file = codecs.open(STAR_FILE, 'r', 'utf-8') file_content = file.read() file_header = "// File : {0}".format(STAR_FILE) self.not_understood_content = u"{0}\n\n{1}".format(file_header, file_content) except: self.not_understood_content = u"Error while reading file '{0}'. Error is : {1}".format(STAR_FILE, traceback.format_exc()) self.log.error(self.not_understood_content) else: self.not_understood_content = u"" self.log.info(u"Not understood queries file NOT found : {0}. This is not an error. Your butler is just awesome (or unused) ;)".format(STAR_FILE))
def __init__(self): pub = MQPub(zmq.Context(), 'example-pub') pub.send_event('client.sensor', \ {"12324" : 678, "3456" : 9999, "atTimestamp" : "1234567890"})
class Interface(Plugin, MQAsyncSub): ''' For interfaces clients ''' def __init__(self, name, stop_cb = None, is_manager = False, parser = None, daemonize = True, log_prefix = "interface_", log_on_stdout = True, test = False, source = None): ''' Create Interface instance, which defines system handlers @param source : overwrite the source value (client-device.instance) ''' Plugin.__init__(self, name, type = "interface", stop_cb = stop_cb, is_manager = is_manager, parser = parser, daemonize = daemonize, log_prefix = log_prefix, log_on_stdout = log_on_stdout, test = test) self.log.info(u"Start of the interface init") # define the source (in can be used in some plugins) if source == None: self.source = "{0}-{1}.{2}".format(INTERFACE_VENDOR_ID, self.get_plugin_name(), self.get_sanitized_hostname()) # in case we overwrite the source : else: self.source = source ### MQ self._mq_name = self.source #self.zmq = zmq.Context() self.mq_pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs MQAsyncSub.__init__(self, self.zmq, self._name, ['interface.output']) ### Context # set the context # All elements that may be added in the request sent over MQ # * media (irc, audio, sms, ...) # * text (from voice recognition) # * location (the input element location : this is configured on the input element : kitchen, garden, bedroom, ...) # * identity (from face recognition) # * mood (from kinect or anything else) # * sex (from voice recognition and post processing on the voice) self.butler_context = {"media" : None, "location" : None, "identity" : None, "mood" : None, "sex" : None } self.log.info(u"End of the interface init") def send_to_butler(self, message, identity = None, media = None, location = None, sex = None, mood = None): """ Send a message over MQ to the butler component """ self.log.info(u"Send a message to the butler : {0}".format(message)) request = self.butler_context request["text"] = message request["source"] = self.source if identity: request["identity"] = identity if media: request["media"] = media if location: request["location"] = location if sex: request["sex"] = sex if mood: request["mood"] = mood self.mq_pub.send_event('interface.input', request) def on_message(self, msgid, content): """ When a message is received from the MQ (pub/sub) """ if msgid == "interface.output": self.log.debug(u"interface.output received message : {0}".format(content)) self.process_response(content) return # TODO : handle context ### filter on location, media # if media not cli, don't process if content['media'] != "cli": return # location # no location, so no check ### Display message in the cli #print("{0}".format(content['text'])) print(u"{0} > {1}".format(BUTLER_CLI_NAME, content['text'])) ### Display the prompt for the next exchange print(u"{0} > ".format(BUTLER_CLI_USER)) def process_response(self, data): """ empty function that should be override to process the butler response """ pass
def __init__(self, name, type = "plugin", stop_cb = None, is_manager = False, parser = None, daemonize = True, log_prefix = "", log_on_stdout = True, test = False): ''' Create Plugin instance, which defines system handlers @param name : The name of the current client @param type : The type of the current client (default = 'plugin' for xpl plugins @param stop_cb : Additionnal method to call when a stop request is received @param is_manager : Must be True if the child script is a Domogik Manager process You should never need to set it to True unless you develop your own manager @param parser : An instance of ArgumentParser. If you want to add extra options to the generic option parser, create your own ArgumentParser instance, use parser.add_argument and then pass your parser instance as parameter. Your options/params will then be available on self.options and self.args @param daemonize : If set to False, force the instance *not* to daemonize, even if '-f' is not passed on the command line. If set to True (default), will check if -f was added. @param log_prefix : If set, use this prefix when creating the log file in Logger() @param log_on_stdout : If set to True, allow to read the logs on both stdout and log file ''' BasePlugin.__init__(self, name, stop_cb, parser, daemonize, log_prefix, log_on_stdout) Watcher(self) self.log.info(u"----------------------------------") self.log.info(u"Starting client '{0}' (new manager instance)".format(name)) self.log.info(u"Python version is {0}".format(sys.version_info)) if self.options.test_option: self.log.info(u"The client is starting in TEST mode. Test option is {0}".format(self.options.test_option)) self._type = type self._name = name self._test = test # flag used to avoid loading json in test mode ''' Calculate the MQ name - For a core component this is just its component name (self._name) - For a client this is <self._type>-<self._name>-self.hostname The reason is that the core components need a fixed name on the mq network, if a client starts up it needs to request the config on the network, and it needs to know the worker (core component) to ask the config from. Because of the above reason, every item in the core_component list can only run once ''' if self._name in CORE_COMPONENTS: self._mq_name = self._name else: self._mq_name = "{0}-{1}.{2}".format(self._type, self._name, self.get_sanitized_hostname()) # MQ publisher and REP self.zmq = zmq.Context() self._pub = MQPub(self.zmq, self._mq_name) self._set_status(STATUS_STARTING) # MQ : start the thread which sends the status each N seconds thr_send_status = threading.Thread(None, self._send_status_loop, "send_status_loop", (), {}) thr_send_status.start() ### MQ # for stop requests MQRep.__init__(self, self.zmq, self._mq_name) self.helpers = {} self._is_manager = is_manager cfg = Loader('domogik') my_conf = cfg.load() self._config_files = CONFIG_FILE self.config = dict(my_conf[1]) self.libraries_directory = self.config['libraries_path'] self.packages_directory = "{0}/{1}".format(self.config['libraries_path'], PACKAGES_DIR) self.resources_directory = "{0}/{1}".format(self.config['libraries_path'], RESOURCES_DIR) self.products_directory = "{0}/{1}_{2}/{3}".format(self.packages_directory, self._type, self._name, PRODUCTS_DIR) # client config self._client_config = None # Get pid and write it in a file self._pid_dir_path = self.config['pid_dir_path'] self._get_pid() if len(self.get_sanitized_hostname()) > 16: self.log.error(u"You must use 16 char max hostnames ! {0} is {1} long".format(self.get_sanitized_hostname(), len(self.get_sanitized_hostname()))) self.force_leave() return # Create object which get process informations (cpu, memory, etc) # TODO : activate # TODO : use something else that xPL ????????? #self._process_info = ProcessInfo(os.getpid(), # TIME_BETWEEN_EACH_PROCESS_STATUS, # self._send_process_info, # self.log, # self.myxpl) #self._process_info.start() self.dont_run_ready = False # for all no core elements, load the json # TODO find a way to do it nicer ?? if self._name not in CORE_COMPONENTS and self._test == False: self._load_json() # init an empty devices list self.devices = [] # init an empty 'new' devices list self.new_devices = [] # check for products pictures if self._name not in CORE_COMPONENTS and self._test == False: self.check_for_pictures() # init finished self.log.info(u"End init of the global client part")
class Butler(Plugin): """ Butler component TODO : * /quit /reload commands * interact with domogik : commands """ def __init__(self): ### Option parser parser = ArgumentParser() parser.add_argument("-i", action="store_true", dest="interactive", default=False, \ help="Butler interactive mode (must be used WITH -f).") Plugin.__init__(self, name='butler', parser=parser, log_prefix='core_') ### MQ # MQ publisher #self._mq_name = "interface-{0}.{1}".format(self._name, self.get_sanitized_hostname()) self._mq_name = "butler" #self.zmq = zmq.Context() self.pub = MQPub(self.zmq, self._mq_name) # subscribe the MQ for interfaces inputs self.add_mq_sub('interface.input') # devices updates self.add_mq_sub('device.update') ### Configuration elements try: cfg = Loader('butler') config = cfg.load() conf = dict(config[1]) self.lang = conf['lang'] self.butler_name = conf['name'] self.log.debug(u"The butler configured name is '{0}'".format( self.butler_name)) self.butler_name_cleaned = clean_input(conf['name']) self.log.debug(u"The butler cleaned name is '{0}'".format( self.butler_name_cleaned)) self.butler_sex = conf['sex'] self.butler_mood = None if self.butler_sex not in SEX_ALLOWED: self.log.error( u"Exiting : the butler sex configured is not valid : '{0}'. Expecting : {1}" .format(self.butler_sex, SEX_ALLOWED)) self.force_leave() return except: self.log.error( u"Exiting : error while reading the configuration file '{0}' : {1}" .format(CONFIG_FILE, traceback.format_exc())) self.force_leave() return # user name (default is 'localuser') # this is not used for now on Domogik side self.user_name = "localuser" ### Prepare the brain # - validate packages # Start the brain :) self.brain = RiveScript(utf8=True) # set rivescript variables # Configure bot variables # all must be lower case.... self.log.info(u"Configuring name and sex : {0}, {1}".format( self.butler_name_cleaned.lower(), self.butler_sex.lower())) self.brain.set_variable(u"name", self.butler_name_cleaned.lower()) self.brain.set_variable(u"fullname", self.butler_name.lower()) self.brain.set_variable(u"sex", self.butler_sex.lower()) # set the PYTHONPATH sys.path.append(self.get_libraries_directory()) # load the brain self.brain_content = None self.learn_content = None self.not_understood_content = None self.load_all_brain() # shortcut to allow the core brain package to reload the brain for learning self.brain.reload_butler = self.reload # shortcut to allow the core brain package to do logging and access the devices in memory self.brain.log = self.log self.brain.devices = [] # will be loaded in self.reload_devices() # history self.history = [] # load all known devices self.reload_devices() self.log.info( u"*** Welcome in {0} world, your digital assistant! ***".format( self.butler_name)) # for chat more only #self.log.info(u"You may type /quit to let {0} have a break".format(self.butler_name)) ### Interactive mode if self.options.interactive: self.log.info(u"Launched in interactive mode : running the chat!") # TODO : run as a thread #self.run_chat() thr_run_chat = Thread(None, self.run_chat, "run_chat", (), {}) thr_run_chat.start() else: self.log.info(u"Not launched in interactive mode") ### TODO #self.add_stop_cb(self.shutdown) self.log.info(u"Butler initialized") self.ready() def reload_devices(self): """ Load or reload the devices list in memory to improve the butler speed (mainly for brain-base package usage) """ self.log.info(u"Request the devices list over MQ...") try: cli = MQSyncReq(zmq.Context()) msg = MQMessage() msg.set_action('device.get') str_devices = cli.request('dbmgr', msg.get(), timeout=10).get()[1] self.devices = json.loads(str_devices)['devices'] self.log.info(u"{0} devices loaded!".format(len(self.devices))) except: self.log.error( u"Error while getting the devices list over MQ. Error is : {0}" .format(traceback.format_exc())) self.devices = [] self.brain.devices = self.devices def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ try: ### discuss over Req/Rep (used by rest url) if msg.get_action() == "butler.discuss.do": self.log.info(u"Discuss request : {0}".format(msg)) self._mdp_reply_butler_discuss(msg) ### rivescript files detail elif msg.get_action() == "butler.scripts.get": self.log.info(u"Scripts request : {0}".format(msg)) self._mdp_reply_butler_scripts(msg) ### rivescript files detail elif msg.get_action() == "butler.reload.do": self.log.info(u"Reload brain request : {0}".format(msg)) self._mdp_reply_butler_reload(msg) ### history elif msg.get_action() == "butler.history.get": self.log.info(u"Get butler history : {0}".format(msg)) self._mdp_reply_butler_history(msg) ### features elif msg.get_action() == "butler.features.get": self.log.info(u"Get butler features : {0}".format(msg)) self._mdp_reply_butler_features(msg) except: self.log.error( u"Error while processing MQ message : '{0}'. Error is : {1}". format(msg, traceback.format_exc())) def _mdp_reply_butler_discuss(self, message): """ Discuss over req/rep this should NOT be called with a 10 seconds timeout... """ # TODO : merge with the on_message function !!! content = message.get_data() self.log.info(u"Received message : {0}".format(content)) self.add_to_history(u"interface.input", content) reply = self.process(content['text']) # fill empty data for elt in [ 'identity', 'media', 'location', 'sex', 'mood', 'reply_to' ]: if elt not in content: content[elt] = None # publish over MQ data = { "media": content['media'], "location": content['location'], "sex": self.butler_sex, "mood": self.butler_mood, "is_reply": True, "reply_to": content['source'], "identity": self.butler_name, "lang": self.lang, "text": reply } self.log.info(u"Send response over MQ : {0}".format(data)) msg = MQMessage() msg.set_action('butler.discuss.result') msg.set_data(data) self.reply(msg.get()) self.add_to_history(u"interface.output", data) def _mdp_reply_butler_scripts(self, message): """ Send the raw content for the brain parts over the MQ """ # TODO : handle choice of the client in the req message # load not understood queries data self.read_not_understood_file() msg = MQMessage() msg.set_action('butler.scripts.result') msg.add_data(u"learn", self.learn_content) msg.add_data(u"not_understood", self.not_understood_content) for client_id in self.brain_content: msg.add_data(client_id, self.brain_content[client_id]) self.reply(msg.get()) def _mdp_reply_butler_reload(self, message): """ Reload the brain """ msg = MQMessage() msg.set_action('butler.reload.result') try: self.reload() msg.add_data(u"status", True) msg.add_data(u"reason", "") except: msg.add_data(u"status", False) msg.add_data( u"reason", "Error while reloading brain parts : {0}".format( traceback.format_exc())) self.reply(msg.get()) def _mdp_reply_butler_history(self, message): """ Butler history """ msg = MQMessage() msg.set_action('butler.history.result') msg.add_data(u"history", self.history) self.reply(msg.get()) def _mdp_reply_butler_features(self, message): """ Butler features """ msg = MQMessage() msg.set_action('butler.features.result') msg.add_data(u"features", self.butler_features) self.reply(msg.get()) def reload(self): self.load_all_brain() def load_all_brain(self): """ Load all the brain parts (included in domogik or in packages) and do any other related actions """ try: # load the minimal brain self.brain_content = { } # the brain raw content for display in the admin (transmitted over MQ) self.log.info(u"Load minimal brain : {0}".format(MINIMAL_BRAIN)) self.brain.load_file(MINIMAL_BRAIN) # load packages for the brain self.load_brain_parts() # sort replies self.brain.sort_replies() except: self.log.error(u"Error while loading brain : {0}".format( traceback.format_exc())) def load_brain_parts(self): """ Load the parts of the brain from /var/lib/domogik/domogik_packages/brain_* and also plugin_* because some plugins may need dedicated brain parts : - weather forecast - anything less generic than a datatype basic usage """ try: list = [] # first load the packages parts dir_list = os.listdir(self.get_packages_directory()) # first, make sure the brain_base package is loaded the first!! if BRAIN_BASE in dir_list: dir_list.remove(BRAIN_BASE) dir_list = [BRAIN_BASE] + dir_list for a_file in dir_list: try: pkg_type, name = a_file.split("_") except ValueError: # not a foo_bar file : skip it continue #if a_file[0:len(BRAIN_PKG_TYPE)] == BRAIN_PKG_TYPE: if pkg_type in BRAIN_PKG_TYPES: client_id = "{0}-{1}.{2}".format( pkg_type, a_file.split("_")[1], self.get_sanitized_hostname()) self.brain_content[client_id] = {} rs_dir = os.path.join(self.get_packages_directory(), a_file, RIVESCRIPT_DIR) if os.path.isdir(rs_dir): self.log.info(u"Brain part found : {0}".format(a_file)) #self.log.debug(u"The brain part contains a rivescript folder ({0})".format(RIVESCRIPT_DIR)) lang_dir = os.path.join(rs_dir, self.lang) if os.path.isdir(lang_dir): self.log.info(u"- Language found : {0}".format( self.lang)) # add the brain part to rivescript self.brain.load_directory(lang_dir) # add the files raw data to brain content (to be sent over MQ to the admin) self.brain_content[client_id][self.lang] = {} for a_rs_file in os.listdir(lang_dir): a_rs_file_path = os.path.join( lang_dir, a_rs_file) if os.path.isfile( a_rs_file_path) and a_rs_file[ -len(RIVESCRIPT_EXTENSION ):] == RIVESCRIPT_EXTENSION: try: import codecs file = codecs.open( a_rs_file_path, 'r', 'utf-8') file_content = file.read() content = u"{0}".format(file_content) except: content = u"Error while reading file '{0}'. Error is : {1}".format( a_rs_file_path, traceback.format_exc()) self.log.error(content) self.brain_content[client_id][ self.lang][a_rs_file] = content # and finally, load the learning file # this file is generated over the time when the domogik.butler.brain learn() function is called learn_file = LEARN_FILE if os.path.isfile(learn_file): self.log.info(u"Learn file found : {0}".format(learn_file)) # add the brain part to rivescript self.brain.load_file(learn_file) try: import codecs file = codecs.open(learn_file, 'r', 'utf-8') file_content = file.read() file_header = "// File : {0}".format(learn_file) self.learn_content = u"{0}\n\n{1}".format( file_header, file_content) except: self.learn_content = u"Error while reading file '{0}'. Error is : {1}".format( learn_file, traceback.format_exc()) self.log.error(self.learn_content) else: self.learn_content = u"" self.log.info( u"Learn file NOT found : {0}. This is not an error. You just have learn nothing to your butler ;)" .format(learn_file)) # to finish, find all the tagged features # and all the tagged suggestions self.get_brain_features_and_suggestions() except: msg = "Error accessing packages directory : {0}. You should create it".format( str(traceback.format_exc())) self.log.error(msg) def get_brain_features_and_suggestions(self): """ Extract brain features and suggestions from the rivescript files : // ##feature## a feature + feature trigger - feature response /* ##suggest## ? ... @ ... */ """ self.butler_features = [] self.butler_suggestions = [] try: self.log.info( u"Extract tagged features (##feature##) and suggestions (##suggest##) from the rivescript files" ) for client in self.brain_content: for lang in self.brain_content[client]: for fic in self.brain_content[client][lang]: the_suggests = re.findall( SUGGEST_REGEXP, self.brain_content[client][lang][fic]) if the_suggests != []: self.butler_suggestions.extend(the_suggests) for line in self.brain_content[client][lang][ fic].split("\n"): if re.search(FEATURE_TAG, line): self.butler_features.append( line.split(FEATURE_TAG)[1]) self.log.info(u"{0} feature(s) found".format( len(self.butler_features))) self.log.info(u"{0} suggestion(s) found".format( len(self.butler_suggestions))) # store in the Rivescript object the features and suggestions to be able to grab them from the core brain package self.brain.the_features = '.\n'.join(self.butler_features) self.brain.the_suggestions = self.butler_suggestions except: self.log.error(u"Error while extracting the features : {0}".format( traceback.format_exc())) def process(self, query): """ Process the input query by calling rivescript brain @param query : the text query """ reply = "" try: self.log.debug(u"Before transforming query : {0}".format(query)) self.brain.raw_query = query query = clean_input(query) self.log.debug(u"After transforming query : {0}".format(query)) self.brain.query = query # process the query self.log.debug( u"Before calling Rivescript brain for processing : {0} (type={1})" .format(query, type(query))) reply = self.brain.reply(self.user_name, query) self.log.debug( u"Processing finished. The reply is : {0}".format(reply)) return reply except: self.log.error( u"Error while processing query '{0}'. Error is : {1}".format( query, traceback.format_exc())) return "Error" def shutdown(self): """ Shutdown the butler """ pass def add_to_history(self, msgid, data): self.history.append({"msgid": msgid, "context": data}) def on_message(self, msgid, content): """ When a message is received from the MQ (pub/sub) """ if msgid == "device.update": self.reload_devices() elif msgid == "interface.input": # TODO : # merge with the on_mdp_reply_butler_disscuss() function self.log.info(u"Received message : {0}".format(content)) ### Get response from the brain # TODO : do it in a thread and if this last too long, do : # 3s : reply "humm..." # 10s : reply "I am checking..." # 20s : reply "It takes already 20s for processing, I cancel the request" and kill the thread #reply = self.brain.reply(self.user_name, content['text']) self.add_to_history(u"interface.input", content) reply = self.process(content['text']) ### Prepare response for the MQ # All elements that may be added in the request sent over MQ for interface.output # * media (irc, audio, sms, ...) # * text (from voice recognition) # * location (the input element location : this is configured on the input element : kitchen, garden, bedroom, ...) # * reply_to (from face recognition) #self.context = {"media" : "irc", # "location" : "internet", # "reply_to" : content['identity'] # } #self.context['text'] = reply # fill empty data for elt in [ 'identity', 'media', 'location', 'sex', 'mood', 'reply_to' ]: if elt not in content: content[elt] = None # publish over MQ data = { "media": content['media'], "location": content['location'], "sex": self.butler_sex, "mood": self.butler_mood, "is_reply": True, "reply_to": content['source'], "identity": self.butler_name, "lang": self.lang, "text": reply } self.log.info(u"Send response over MQ : {0}".format(data)) self.pub.send_event('interface.output', data) self.add_to_history(u"interface.output", data) def run_chat(self): """ Nestor starts to serve his master over the chat the chat is really usefull for debugging TODO : allow Nestor to connect over irc on demand for test purpose """ # just wait for 2s to have a cleaner output #time.sleep(2) # start serving for an entire life while True: msg = raw_input(u"You > ") # first, let python handle some system messages if msg == '/quit': quit() # then, let Nestor do his work!!! #reply = self.brain.reply(self.user_name, msg) reply = self.process(msg) # let Nestor answer in the chat print(u"{0} > {1}".format(ucode(self.butler_name), ucode(reply))) # let Nestor speak #tts = u"espeak -p 40 -s 140 -v mb/mb-fr1 \"{0}\" | mbrola /usr/share/mbrola/fr1/fr1 - -.au | aplay".format(reply) #tts = u"espeak -p 40 -s 140 -v mb/mb-fr1 \"{0}\" | mbrola /usr/share/mbrola/fr1 - -.au | aplay".format(reply) #subp = Popen(tts, shell=True) #pid = subp.pid #subp.communicate() def read_not_understood_file(self): """ Get the content of the non understood queries file """ if os.path.isfile(STAR_FILE): self.log.info( u"Not understood queries file found : {0}".format(STAR_FILE)) try: import codecs file = codecs.open(STAR_FILE, 'r', 'utf-8') file_content = file.read() file_header = "// File : {0}".format(STAR_FILE) self.not_understood_content = u"{0}\n\n{1}".format( file_header, file_content) except: self.not_understood_content = u"Error while reading file '{0}'. Error is : {1}".format( STAR_FILE, traceback.format_exc()) self.log.error(self.not_understood_content) else: self.not_understood_content = u"" self.log.info( u"Not understood queries file NOT found : {0}. This is not an error. Your butler is just awesome (or unused) ;)" .format(STAR_FILE))
class Plugin(BasePlugin, MQRep): ''' Global plugin class, manage signal handlers. This class shouldn't be used as-it but should be extended by no xPL plugin or by the class xPL plugin which will be used by the xPL plugins This class is a Singleton Please keep in mind that the name 'Plugin' is historical. This class is here the base class to use for all kind of clients : plugin (xpl plugin, interface, ...) ''' def __init__(self, name, type = "plugin", stop_cb = None, is_manager = False, parser = None, daemonize = True, log_prefix = "", log_on_stdout = True, test = False): ''' Create Plugin instance, which defines system handlers @param name : The name of the current client @param type : The type of the current client (default = 'plugin' for xpl plugins @param stop_cb : Additionnal method to call when a stop request is received @param is_manager : Must be True if the child script is a Domogik Manager process You should never need to set it to True unless you develop your own manager @param parser : An instance of ArgumentParser. If you want to add extra options to the generic option parser, create your own ArgumentParser instance, use parser.add_argument and then pass your parser instance as parameter. Your options/params will then be available on self.options and self.args @param daemonize : If set to False, force the instance *not* to daemonize, even if '-f' is not passed on the command line. If set to True (default), will check if -f was added. @param log_prefix : If set, use this prefix when creating the log file in Logger() @param log_on_stdout : If set to True, allow to read the logs on both stdout and log file ''' BasePlugin.__init__(self, name, stop_cb, parser, daemonize, log_prefix, log_on_stdout) Watcher(self) self.log.info(u"----------------------------------") self.log.info(u"Starting client '{0}' (new manager instance)".format(name)) self.log.info(u"Python version is {0}".format(sys.version_info)) if self.options.test_option: self.log.info(u"The client is starting in TEST mode. Test option is {0}".format(self.options.test_option)) self._type = type self._name = name self._test = test # flag used to avoid loading json in test mode ''' Calculate the MQ name - For a core component this is just its component name (self._name) - For a client this is <self._type>-<self._name>-self.hostname The reason is that the core components need a fixed name on the mq network, if a client starts up it needs to request the config on the network, and it needs to know the worker (core component) to ask the config from. Because of the above reason, every item in the core_component list can only run once ''' if self._name in CORE_COMPONENTS: self._mq_name = self._name else: self._mq_name = "{0}-{1}.{2}".format(self._type, self._name, self.get_sanitized_hostname()) # MQ publisher and REP self.zmq = zmq.Context() self._pub = MQPub(self.zmq, self._mq_name) self._set_status(STATUS_STARTING) # MQ : start the thread which sends the status each N seconds thr_send_status = threading.Thread(None, self._send_status_loop, "send_status_loop", (), {}) thr_send_status.start() ### MQ # for stop requests MQRep.__init__(self, self.zmq, self._mq_name) self.helpers = {} self._is_manager = is_manager cfg = Loader('domogik') my_conf = cfg.load() self._config_files = CONFIG_FILE self.config = dict(my_conf[1]) self.libraries_directory = self.config['libraries_path'] self.packages_directory = "{0}/{1}".format(self.config['libraries_path'], PACKAGES_DIR) self.resources_directory = "{0}/{1}".format(self.config['libraries_path'], RESOURCES_DIR) self.products_directory = "{0}/{1}_{2}/{3}".format(self.packages_directory, self._type, self._name, PRODUCTS_DIR) # client config self._client_config = None # Get pid and write it in a file self._pid_dir_path = self.config['pid_dir_path'] self._get_pid() if len(self.get_sanitized_hostname()) > 16: self.log.error(u"You must use 16 char max hostnames ! {0} is {1} long".format(self.get_sanitized_hostname(), len(self.get_sanitized_hostname()))) self.force_leave() return # Create object which get process informations (cpu, memory, etc) # TODO : activate # TODO : use something else that xPL ????????? #self._process_info = ProcessInfo(os.getpid(), # TIME_BETWEEN_EACH_PROCESS_STATUS, # self._send_process_info, # self.log, # self.myxpl) #self._process_info.start() self.dont_run_ready = False # for all no core elements, load the json # TODO find a way to do it nicer ?? if self._name not in CORE_COMPONENTS and self._test == False: self._load_json() # init an empty devices list self.devices = [] # init an empty 'new' devices list self.new_devices = [] # check for products pictures if self._name not in CORE_COMPONENTS and self._test == False: self.check_for_pictures() # init finished self.log.info(u"End init of the global client part") def check_configured(self): """ For a client only To be call in the client __init__() Check in database (over queryconfig) if the key 'configured' is set to True for the client if not, stop the client and log this """ self._client_config = Query(self.zmq, self.log) configured = self._client_config.query(self._type, self._name, 'configured') if configured == '1': configured = True if configured != True: self.log.error(u"The client is not configured (configured = '{0}'. Stopping the client...".format(configured)) self.force_leave(status = STATUS_NOT_CONFIGURED) return False self.log.info(u"The client is configured. Continuing (hoping that the user applied the appropriate configuration ;)") return True def _load_json(self): """ Load the client json file """ try: self.log.info(u"Read the json file and validate id".format(self._name)) pkg_json = PackageJson(pkg_type = self._type, name = self._name) # check if json is valid if pkg_json.validate() == False: # TODO : how to get the reason ? self.log.error(u"Invalid json file") self.force_leave(status = STATUS_INVALID) else: # if valid, store the data so that it can be used later self.log.info(u"The json file is valid") self.json_data = pkg_json.get_json() except: self.log.error(u"Error while trying to read the json file : {1}".format(self._name, traceback.format_exc())) self.force_leave(status = STATUS_INVALID) def get_config(self, key): """ Try to get the config over the MQ. If value is None, get the default value """ if self._client_config == None: self._client_config = Query(self.zmq, self.log) value = self._client_config.query(self._type, self._name, key) if value == None or value == 'None': self.log.info(u"Value for '{0}' is None or 'None' : trying to get the default value instead...".format(key)) value = self.get_config_default_value(key) self.log.info(u"Value for '{0}' is : {1}".format(key, value)) return self.cast_config_value(key, value) def get_config_default_value(self, key): """ Get the default value for a config key from the json file @param key : configuration key """ for idx in range(len(self.json_data['configuration'])): if self.json_data['configuration'][idx]['key'] == key: default = self.json_data['configuration'][idx]['default'] self.log.info(u"Default value required for key '{0}' = {1}".format(key, default)) return default def cast_config_value(self, key, value): """ Cast the config value as the given type in the json file @param key : configuration key @param value : configuration value to cast and return @return : the casted value """ for idx in range(len(self.json_data['configuration'])): if self.json_data['configuration'][idx]['key'] == key: type = self.json_data['configuration'][idx]['type'] self.log.info(u"Casting value for key '{0}' in type '{1}'...".format(key, type)) cvalue = self.cast(value, type) self.log.info(u"Value is : {0}".format(cvalue)) return cvalue # no cast operation : return the value if value == "None": return None return value def cast(self, value, type): """ Cast a value for a type @param value : value to cast @param type : type in which you want to cast the value """ try: if type == "boolean": # just in case, the "True"/"False" are not already converted in True/False # this is (currently) done on queryconfig side if value in ["True", "Y"]: return True elif value in ["False", "N"]: return False # type == choice : nothing to do if type == "date": self.log.error(u"TODO : the cast in date format is not yet developped. Please request fritz_smh to do it") if type == "datetime": self.log.error(u"TODO : the cast in date format is not yet developped. Please request fritz_smh to do it") # type == email : nothing to do if type == "float": return float(value) if type == "integer": return float(value) # type == ipv4 : nothing to do # type == multiple choice : nothing to do # type == string : nothing to do if type == "time": self.log.error(u"TODO : the cast in date format is not yet developped. Please request fritz_smh to do it") # type == url : nothing to do except: # if an error occurs : return the default value and log a warning self.log.warning(u"Error while casting value '{0}' to type '{1}'. The client may not work!! Error : {2}".format(value, type, traceback.format_exc())) return value return value def get_device_list(self, quit_if_no_device = False, max_attempt = 2): """ Request the dbmgr component over MQ to get the devices list for this client @param quit_if_no_device: if True, exit the client if there is no devices or MQ request fail @param max_attempt : number of retry MQ request if it fail """ self.log.info(u"Retrieve the devices list for this client...") msg = MQMessage() msg.set_action('device.get') msg.add_data('type', self._type) msg.add_data('name', self._name) msg.add_data('host', self.get_sanitized_hostname()) attempt = 1 result = None while not result and attempt <= max_attempt : mq_client = MQSyncReq(self.zmq) result = mq_client.request('dbmgr', msg.get(), timeout=10) if not result: self.log.warn(u"Unable to retrieve the device list (attempt {0}/{1})".format(attempt, max_attempt)) attempt += 1 if not result: self.log.error(u"Unable to retrieve the device list, max attempt achieved : {0}".format(max_attempt)) if quit_if_no_device: self.log.warn(u"The developper requested to stop the client if error on retrieve the device list") self.force_leave() return [] else: device_list = result.get_data()['devices'] if device_list == []: self.log.warn(u"There is no device created for this client") if quit_if_no_device: self.log.warn(u"The developper requested to stop the client if there is no device created") self.force_leave() return [] for a_device in device_list: self.log.info(u"- id : {0} / name : {1} / device type id : {2}".format(a_device['id'], \ a_device['name'], \ a_device['device_type_id'])) # log some informations about the device # notice that even if we are not in the XplPlugin class we will display xpl related informations : # for some no xpl clients, there will just be nothing to display. # first : the stats self.log.info(u" xpl_stats features :") for a_xpl_stat in a_device['xpl_stats']: self.log.info(u" - {0}".format(a_xpl_stat)) self.log.info(u" Static Parameters :") for a_feature in a_device['xpl_stats'][a_xpl_stat]['parameters']['static']: self.log.info(u" - {0} = {1}".format(a_feature['key'], a_feature['value'])) self.log.info(u" Dynamic Parameters :") for a_feature in a_device['xpl_stats'][a_xpl_stat]['parameters']['dynamic']: self.log.info(u" - {0}".format(a_feature['key'])) # then, the commands self.log.info(u" xpl_commands features :") for a_xpl_cmd in a_device['xpl_commands']: self.log.info(u" - {0}".format(a_xpl_cmd)) self.log.info(u" + Parameters :") for a_feature in a_device['xpl_commands'][a_xpl_cmd]['parameters']: self.log.info(u" - {0} = {1}".format(a_feature['key'], a_feature['value'])) self.devices = device_list return device_list def device_detected(self, data): """ The clients developpers can call this function when a device is detected This function will check if a corresponding device exists and : - if so, do nothing - if not, add the device in a 'new devices' list - if the device is already in the 'new devices list', does nothing - if not : add it into the list and send a MQ message : an event for the UI to say a new device is detected @param data : data about the device Data example : { "device_type" : "...", "reference" : "...", "global" : [ { "key" : "....", "value" : "...." }, ... ], "xpl" : [ { "key" : "....", "value" : "...." }, ... ], "xpl_commands" : { "command_id" : [ { "key" : "....", "value" : "...." }, ... ], "command_id_2" : [...] }, "xpl_stats" : { "sensor_id" : [ { "key" : "....", "value" : "...." }, ... ], "sensor_id_2" : [...] } } """ self.log.debug(u"Device detected : data = {0}".format(data)) # browse all devices to find if the device exists found = False for a_device in self.devices: # filter on appropriate device_type if a_device['device_type_id'] != data['device_type']: continue # handle "main" global parameters # TODO ???? # handle xpl global parameters if data['xpl'] != []: for dev_feature in a_device['xpl_stats']: for dev_param in a_device['xpl_stats'][dev_feature]['parameters']['static']: #print(dev_param) for found_param in data['xpl']: if dev_param['key'] == found_param['key'] and dev_param['value'] == found_param['value']: found = True #print("FOUND") break for dev_feature in a_device['xpl_commands']: for dev_param in a_device['xpl_commands'][dev_feature]['parameters']['static']: #print(dev_param) for found_param in data['xpl']: if dev_param['key'] == found_param['key'] and dev_param['value'] == found_param['value']: found = True #print("FOUND") break # handle xpl specific parameters if not found and data['xpl_stats'] != []: for dev_feature in a_device['xpl_stats']: for dev_param in a_device['xpl_stats'][dev_feature]['parameters']['static']: #print(dev_param) for found_param in data['xpl_stats']: if dev_param['key'] == found_param['key'] and dev_param['value'] == found_param['value']: found = True #print("FOUND") break if not found and data['xpl_commands'] != []: for dev_feature in a_device['xpl_commands']: for dev_param in a_device['xpl_commands'][dev_feature]['parameters']['static']: #print(dev_param) for found_param in data['xpl_commands']: if dev_param['key'] == found_param['key'] and dev_param['value'] == found_param['value']: found = True #print("FOUND") break if found: self.log.debug(u"The device already exists : id={0}.".format(a_device['id'])) else: self.log.debug(u"The device doesn't exists in database") # generate a unique id for the device from its addresses new_device_id = self.generate_detected_device_id(data) # add the device feature in the new devices list : self.new_devices[device_type][type][feature] = data self.log.debug(u"Check if the device has already be marked as new...") found = False for a_device in self.new_devices: if a_device['id'] == new_device_id: found = True #for a_device in self.new_devices: # if a_device['device_type_id'] == device_type and \ # a_device['type'] == type and \ # a_device['feature'] == feature: # # if data == a_device['data']: # found = True if found == False: new_device = {'id' : new_device_id, 'data' : data} self.log.info(u"New device feature detected and added in the new devices list : {0}".format(new_device)) self.new_devices.append(new_device) # publish new devices update self._pub.send_event('device.new', {"type" : self._type, "name" : self._name, "host" : self.get_sanitized_hostname(), "client_id" : "{0}-{1}.{2}".format(self._type, self._name, self.get_sanitized_hostname()), "device" : new_device}) # TODO : later (0.4.0+), publish one "new device" notification with only the new device detected else: self.log.debug(u"The device has already been detected since the client startup") def generate_detected_device_id(self, data): """ Generate an unique id based on the content of data """ # TODO : improve to make something more sexy ? the_id = json.dumps(data, sort_keys=True) chars_to_remove = ['"', '{', '}', ',', ' ', '=', '[', ']', ':'] the_id = the_id.translate(None, ''.join(chars_to_remove)) return the_id def get_parameter(self, a_device, key): """ For a device feature, return the required parameter value @param a_device: the device informations @param key: the parameter key """ try: self.log.debug(u"Get parameter '{0}'".format(key)) for a_param in a_device['parameters']: if a_param == key: value = self.cast(a_device['parameters'][a_param]['value'], a_device['parameters'][a_param]['type']) self.log.debug(u"Parameter value found: {0}".format(value)) return value self.log.warning(u"Parameter not found : return None") return None except: self.log.error(u"Error while looking for a device parameter. Return None. Error: {0}".format(traceback.format_exc())) return None def get_parameter_for_feature(self, a_device, type, feature, key): """ For a device feature, return the required parameter value @param a_device: the device informations @param type: the parameter type (xpl_stats, ...) @param feature: the parameter feature @param key: the parameter key """ try: self.log.debug(u"Get parameter '{0}' for '{1}', feature '{2}'".format(key, type, feature)) for a_param in a_device[type][feature]['parameters']['static']: if a_param['key'] == key: value = self.cast(a_param['value'], a_param['type']) self.log.debug(u"Parameter value found: {0}".format(value)) return value self.log.warning(u"Parameter not found : return None") return None except: self.log.error(u"Error while looking for a device feature parameter. Return None. Error: {0}".format(traceback.format_exc())) return None def check_for_pictures(self): """ if some products are defined, check if the corresponding pictures are present in the products/ folder """ self.log.info(u"Check if there are pictures for the defined products") ok = True ok_product = None if self.json_data.has_key('products'): for product in self.json_data['products']: ok_product = False for ext in PRODUCTS_PICTURES_EXTENSIONS: file = "{0}.{1}".format(product['id'], ext) if os.path.isfile("{0}/{1}".format(self.get_products_directory(), file)): ok_product = True break if ok_product: self.log.debug(u"- OK : {0} ({1})".format(product['name'], file)) else: ok = False self.log.warning(u"- Missing : {0} ({1}.{2})".format(product['name'], product['id'], PRODUCTS_PICTURES_EXTENSIONS)) if ok == False: self.log.warning(u"Some pictures are missing!") else: if ok_product == None: self.log.info(u"There is no products defined for this client") def ready(self, ioloopstart=1): """ to call at the end of the __init__ of classes that inherits of this one In the XplPLugin class, this function will be completed to also activate the xpl hbeat """ if self.dont_run_ready == True: return ### send client status : STATUS_ALIVE # TODO : why the dbmgr has no self._name defined ??????? # temporary set as unknown to avoir blocking bugs if not hasattr(self, '_name'): self._name = "unknown" self._set_status(STATUS_ALIVE) ### Instantiate the MQ # nothing can be launched after this line (blocking call!!!!) self.log.info(u"Start IOLoop for MQ : nothing else can be executed in the __init__ after this! Make sure that the self.ready() call is the last line of your init!!!!") if ioloopstart == 1: IOLoop.instance().start() def on_mdp_request(self, msg): """ Handle Requests over MQ @param msg : MQ req message """ self.log.debug(u"MQ Request received : {0}" . format(str(msg))) ### stop the client if msg.get_action() == "plugin.stop.do": self.log.info(u"Client stop request : {0}".format(msg)) self._mdp_reply_client_stop(msg) elif msg.get_action() == "helper.list.get": self.log.info(u"Client helper list request : {0}".format(msg)) self._mdp_reply_helper_list(msg) elif msg.get_action() == "helper.help.get": self.log.info(u"Client helper help request : {0}".format(msg)) self._mdp_reply_helper_help(msg) elif msg.get_action() == "helper.do": self.log.info(u"Client helper action request : {0}".format(msg)) self._mdp_reply_helper_do(msg) elif msg.get_action() == "device.new.get": self.log.info(u"Client new devices request : {0}".format(msg)) self._mdp_reply_device_new_get(msg) def _mdp_reply_helper_do(self, msg): contens = msg.get_data() if 'command' in contens.keys(): if contens['command'] in self.helpers.keys(): if 'parameters' not in contens.keys(): contens['parameters'] = {} params = [] else: params = [] for key, value in contens['parameters'].items(): params.append( "{0}='{1}'".format(key, value) ) command = "self.{0}(".format(self.helpers[contens['command']]['call']) command += ", ".join(params) command += ")" result = eval(command) # run the command with all params msg = MQMessage() msg.set_action('helper.do.result') msg.add_data('command', contens['command']) msg.add_data('parameters', contens['parameters']) msg.add_data('result', result) self.reply(msg.get()) def _mdp_reply_helper_help(self, data): content = data.get_data() if 'command' in contens.keys(): if content['command'] in self.helpers.keys(): msg = MQMessage() msg.set_action('helper.help.result') msg.add_data('help', self.helpers[content['command']]['help']) self.reply(msg.get()) def _mdp_reply_client_stop(self, data): """ Stop the client @param data : MQ req message First, send the MQ Rep to 'ack' the request Then, change the client status to STATUS_STOP_REQUEST Then, quit the client by calling force_leave(). This should make the client send a STATUS_STOPPED if all is ok Notice that no check is done on the MQ req content : we need nothing in it as it is directly addressed to a client """ # check if the message is for us content = data.get_data() if content['name'] != self._name or content['host'] != self.get_sanitized_hostname(): return ### Send the ack over MQ Rep msg = MQMessage() msg.set_action('plugin.stop.result') status = True reason = "" msg.add_data('status', status) msg.add_data('reason', reason) msg.add_data('name', self._name) msg.add_data('host', self.get_sanitized_hostname()) self.log.info("Send reply for the stop request : {0}".format(msg)) self.reply(msg.get()) ### Change the client status self._set_status(STATUS_STOP_REQUEST) ### Try to stop the client # if it fails, the manager should try to kill the client self.force_leave() def _mdp_reply_helper_list(self, data): """ Return a list of supported helpers @param data : MQ req message """ ### Send the ack over MQ Rep msg = MQMessage() msg.set_action('helper.list.result') msg.add_data('actions', self.helpers.keys()) self.reply(msg.get()) def _mdp_reply_device_new_get(self, data): """ Return a list of new devices detected @param data : MQ req message """ ### Send the ack over MQ Rep msg = MQMessage() msg.set_action('device.new.result') msg.add_data('devices', self.new_devices) self.reply(msg.get()) def _set_status(self, status): """ Set the client status and send it """ # when ctrl-c is done, there is no more self._name at this point... # why ? because the force_leave method is called twice as show in the logs : # # ^CKeyBoardInterrupt # 2013-12-20 22:48:41,040 domogik-manager INFO Keyboard Interrupt detected, leave now. # Traceback (most recent call last): # File "./manager.py", line 1176, in <module> # main() # File "./manager.py", line 1173, in main # 2013-12-20 22:48:41,041 domogik-manager DEBUG force_leave called # 2013-12-20 22:48:41,044 domogik-manager DEBUG __del__ Single xpl plugin # 2013-12-20 22:48:41,045 domogik-manager DEBUG force_leave called if hasattr(self, '_name'): #if self._name not in CORE_COMPONENTS: # self._status = status # self._send_status() self._status = status self._send_status() def _send_status_loop(self): """ send the status each STATUS_HBEAT seconds """ # TODO : we could optimize by resetting the timer each time the status is sent # but as this is used only to check for dead clients by the manager, it is not very important ;) while not self._stop.isSet(): self._send_status() self._stop.wait(STATUS_HBEAT) def _send_status(self): """ Send the client status over the MQ """ if hasattr(self, "_pub"): if self._name in CORE_COMPONENTS: type = "core" #return else: type = self._type self.log.debug("Send client status : {0}".format(self._status)) self._pub.send_event('plugin.status', {"type" : type, "name" : self._name, "host" : self.get_sanitized_hostname(), "event" : self._status}) def get_config_files(self): """ Return list of config files """ return self._config_files def get_products_directory(self): """ getter """ return self.products_directory def get_libraries_directory(self): """ getter """ return self.libraries_directory def get_packages_directory(self): """ getter """ return self.packages_directory def get_resources_directory(self): """ getter """ return self.resources_directory def get_data_files_directory(self): """ Return the directory where a client developper can store data files. If the directory doesn't exist, try to create it. After that, try to create a file inside it. If something goes wrong, generate an explicit exception. """ path = "{0}/{1}/{2}_{3}/data/".format(self.libraries_directory, PACKAGES_DIR, self._type, self._name) if os.path.exists(path): if not os.access(path, os.W_OK & os.X_OK): raise OSError("Can't write in directory {0}".format(path)) else: try: os.mkdir(path, '0770') self.log.info(u"Create directory {0}.".format(path)) except: raise OSError("Can't create directory {0}.".format(path)) # Commented because : # a write test is done for each call of this function. For a client with a html server (geoloc for example), it # can be an issue as this makes a lot of write for 'nothing' on the disk. # We keep the code for now (0.4) for maybe a later use (and improved) #try: # tmp_prefix = "write_test"; # count = 0 # filename = os.path.join(path, tmp_prefix) # while(os.path.exists(filename)): # filename = "{}.{}".format(os.path.join(path, tmp_prefix),count) # count = count + 1 # f = open(filename,"w") # f.close() # os.remove(filename) #except : # raise IOError("Can't create a file in directory {0}.".format(path)) return path def register_helper(self, action, help_string, callback): if action not in self.helpers: self.helpers[action] = {'call': callback, 'help': help_string} def publish_helper(self, key, data): if hasattr(self, "_pub"): if self._name in CORE_COMPONENTS: type = "core" else: type = self._type self._pub.send_event('helper.publish', {"origin" : self._mq_name, "key": key, "data": data}) def _get_pid(self): """ Get current pid and write it to a file """ pid = os.getpid() pid_file = os.path.join(self._pid_dir_path, self._name + ".pid") self.log.debug(u"Write pid file for pid '{0}' in file '{1}'".format(str(pid), pid_file)) fil = open(pid_file, "w") fil.write(str(pid)) fil.close() def __del__(self): if hasattr(self, "log"): self.log.debug(u"__del__ Single client") self.log.debug(u"the stack is :") for elt in inspect.stack(): self.log.debug(u" {0}".format(elt)) # we guess that if no "log" is defined, the client has not really started, so there is no need to call force leave (and _stop, .... won't be created) self.force_leave() def force_leave(self, status = False, return_code = None): """ Leave threads & timers In the XplPLugin class, this function will be completed to also activate the xpl hbeat """ if hasattr(self, "log"): self.log.debug(u"force_leave called") #self.log.debug(u"the stack is : {0}".format(inspect.stack())) self.log.debug(u"the stack is :") for elt in inspect.stack(): self.log.debug(u" {0}".format(elt)) if return_code != None: self.set_return_code(return_code) self.log.info("Return code set to {0} when calling force_leave()".format(return_code)) # avoid ready() to be launched self.dont_run_ready = True # stop IOLoop #try: # IOLoop.instance().start() #except: # pass # send stopped status over the MQ if status: self._set_status(status) else: self._set_status(STATUS_STOPPED) # try to stop the thread try: self.get_stop().set() except AttributeError: pass if hasattr(self, "_timers"): for t in self._timers: if hasattr(self, "log"): self.log.debug(u"Try to stop timer {0}".format(t)) t.stop() if hasattr(self, "log"): self.log.debug(u"Timer stopped {0}".format(t)) if hasattr(self, "_stop_cb"): for cb in self._stop_cb: if hasattr(self, "log"): self.log.debug(u"Calling stop additionnal method : {0} ".format(cb.__name__)) cb() if hasattr(self, "_threads"): for t in self._threads: if hasattr(self, "log"): self.log.debug(u"Try to stop thread {0}".format(t)) try: t.join() except RuntimeError: pass if hasattr(self, "log"): self.log.debug(u"Thread stopped {0}".format(t)) #t._Thread__stop() #Finally, we try to delete all remaining threads for t in threading.enumerate(): if t != threading.current_thread() and t.__class__ != threading._MainThread: if hasattr(self, "log"): self.log.info(u"The thread {0} was not registered, killing it".format(t.name)) t.join() if hasattr(self, "log"): self.log.info(u"Thread {0} stopped.".format(t.name)) if threading.activeCount() > 1: if hasattr(self, "log"): self.log.warn(u"There are more than 1 thread remaining : {0}".format(threading.enumerate()))