Beispiel #1
0
 def start_stats(self):
     """ Start Statistics manager
     """
     print("Start Stats")
     self.log.info(
         "Starting statistics manager. Its logs will be in a dedicated log file"
     )
     self.stat_mgr = StatsManager(handler_params=[self], xpl=self.myxpl)
     self.stat_mgr.load()
     self.log.info("Stat manager started")
Beispiel #2
0
 def start_stats(self):
     """ Start Statistics manager
     """
     print("Start Stats")
     self.log.info("Starting statistics manager. Its logs will be in a dedicated log file")
     self.stat_mgr = StatsManager(handler_params = [self], xpl = self.myxpl)
     self.stat_mgr.load()
     self.log.info("Stat manager started")
Beispiel #3
0
class Rest(XplPlugin):
    """ REST Server 
        - create a HTTP server 
        - process REST requests
    """
    def __init__(self, server_ip, server_port):
        """ Initiate DbHelper, Logs and config
            Then, start HTTP server and give it initialized data
            @param server_ip :  ip of HTTP server
            @param server_port :  port of HTTP server
        """

        XplPlugin.__init__(self, name='rest')
        # logging initialization
        self.log.info("Rest Server initialisation...")
        self.log.debug("locale : %s %s" % locale.getdefaultlocale())

        # logging Queue activities
        log_queue = logger.Logger('rest-queues')
        self.log_queue = log_queue.get_logger('rest-queues')
        self.log_queue.info("Rest's queues activities...")

        # logging data manipulation initialization
        log_dm = logger.Logger('rest-dm')
        self.log_dm = log_dm.get_logger('rest-dm')
        self.log_dm.info("Rest Server Data Manipulation...")

        # API version
        self._rest_api_version = REST_API_VERSION

        # Hosts list
        self._hosts_list = {
            self.get_sanitized_hostname(): {
                "id": self.get_sanitized_hostname(),
                "status": "on",
                "primary": True,
                "last_seen": time.time(),
                "ip": "",
                "interval": "1"
            }
        }

        try:

            ### Config

            # directory data
            cfg = Loader('domogik')
            config = cfg.load()
            conf = dict(config[1])
            self.log_dir_path = conf['log_dir_path']

            # plugin installation path
            if conf.has_key('package_path'):
                self._package_path = conf['package_path']
                self._src_prefix = None
                self.log.info("Set package path to '%s' " % self._package_path)
                print("Set package path to '%s' " % self._package_path)
                self._design_dir = "%s/domogik_packages/design/" % self._package_path
                self.package_mode = True
            else:
                self.log.info("No package path defined in config file")
                self._package_path = None
                self._src_prefix = conf['src_prefix']
                self._design_dir = "%s/share/domogik/design/" % conf[
                    'src_prefix']
                self.package_mode = False

            # HTTP server ip and port
            try:
                cfg_rest = Loader('rest')
                config_rest = cfg_rest.load()
                conf_rest = dict(config_rest[1])
                self.server_ip = conf_rest['rest_server_ip']
                self.server_port = conf_rest['rest_server_port']
            except KeyError:
                # default parameters
                self.server_ip = server_ip
                self.server_port = server_port
            self.log.info("Configuration : ip:port = %s:%s" %
                          (self.server_ip, self.server_port))

            # SSL configuration
            try:
                cfg_rest = Loader('rest')
                config_rest = cfg_rest.load()
                conf_rest = dict(config_rest[1])
                self.use_ssl = conf_rest['rest_use_ssl']
                if self.use_ssl == "True":
                    self.use_ssl = True
                else:
                    self.use_ssl = False
                self.ssl_certificate = conf_rest['rest_ssl_certificate']
            except KeyError:
                # default parameters
                self.use_ssl = USE_SSL
                self.ssl_certificate = SSL_CERTIFICATE
            if self.use_ssl == True:
                self.log.info(
                    "Configuration : SSL support activated (certificate : %s)"
                    % self.ssl_certificate)
            else:
                self.log.info("Configuration : SSL support not activated")

            # File repository
            try:
                cfg_rest = Loader('rest')
                config_rest = cfg_rest.load()
                conf_rest = dict(config_rest[1])
                self.repo_dir = conf_rest['rest_repository']
            except KeyError:
                # default parameters
                self.repo_dir = DEFAULT_REPO_DIR

            # Gloal Queues config
            self.log.debug("Get queues configuration")
            self._config = Query(self.myxpl, self.log)

            self._queue_timeout = self._config.query('rest', 'q-timeout')
            if self._queue_timeout == None:
                self._queue_timeout = QUEUE_TIMEOUT
            self._queue_timeout = float(self._queue_timeout)

            self._queue_package_size = self._config.query('rest', 'q-pkg-size')
            if self._queue_package_size == None:
                self._queue_package_size = QUEUE_PACKAGE_SIZE
            self._queue_package_size = float(self._queue_package_size)

            self._queue_size = self._config.query('rest', 'q-size')
            if self._queue_size == None:
                self._queue_size = QUEUE_SIZE
            self._queue_size = float(self._queue_size)

            self._queue_life_expectancy = self._config.query(
                'rest', 'q-life-exp')
            if self._queue_life_expectancy == None:
                self._queue_life_expectancy = QUEUE_LIFE_EXPECTANCY
            self._queue_life_expectancy = float(self._queue_life_expectancy)

            self._queue_sleep = self._config.query('rest', 'q-sleep')
            if self._queue_sleep == None:
                self._queue_sleep = QUEUE_SLEEP
            self._queue_sleep = float(self._queue_sleep)

            # /command Queues config
            self._queue_command_size = self._config.query('rest', 'q-cmd-size')
            if self._queue_command_size == None:
                self._queue_command_size = QUEUE_COMMAND_SIZE
            self._queue_command_size = float(self._queue_command_size)

            # /event Queues config
            self._event_timeout = self._config.query('rest', 'evt-timeout')
            if self._event_timeout == None:
                self._event_timeout = EVENT_TIMEOUT
            self._event_timeout = float(self._event_timeout)

            self._queue_event_size = self._config.query('rest', 'q-evt-size')
            if self._queue_event_size == None:
                self._queue_event_size = QUEUE_EVENT_SIZE
            self._queue_event_size = float(self._queue_event_size)

            self._queue_event_timeout = self._config.query(
                'rest', 'q-evt-timeout')
            if self._queue_event_timeout == None:
                self._queue_event_timeout = QUEUE_EVENT_TIMEOUT
            self._queue_event_timeout = float(self._queue_event_timeout)

            self._queue_event_life_expectancy = self._config.query(
                'rest', 'q-evt-life-exp')
            if self._queue_event_life_expectancy == None:
                self._queue_event_life_expectancy = QUEUE_EVENT_LIFE_EXPECTANCY
            self._queue_event_life_expectancy = float(
                self._queue_event_life_expectancy)

            # Queues for xPL
            # Queues for packages management
            self._queue_package = Queue(self._queue_package_size)

            # Queues for domogik system actions
            self._queue_system_list = Queue(self._queue_size)
            self._queue_system_detail = Queue(self._queue_size)
            self._queue_system_start = Queue(self._queue_size)
            self._queue_system_stop = Queue(self._queue_size)

            # Queues for /command
            self._queue_command = Queue(self._queue_command_size)

            # Queues for /events/domogik
            self._queue_event_dmg = Queue(self._queue_event_size)

            # Queues for /events/request
            # this queue will be fill by stat manager
            self._event_requests = RequestEvents(
                self.get_stop, self.log, self._event_timeout,
                self._queue_event_size, self._queue_event_timeout,
                self._queue_event_life_expectancy)
            self.add_stop_cb(self._event_requests.set_stop_clean)

            # Queues for /events/domogik
            # this queue will be fill by stat manager
            self._event_dmg = DmgEvents(self.get_stop, self.log,
                                        self._event_timeout,
                                        self._queue_event_size,
                                        self._queue_event_timeout,
                                        self._queue_event_life_expectancy)
            # notice : adding data in queue is made in _add_to_queue_system_list
            self.add_stop_cb(self._event_dmg.set_stop_clean)

            # define listeners for queues
            self.log.debug("Create listeners")
            if self.package_mode == True:
                Listener(self._list_installed_packages, self.myxpl, \
                         {'schema': 'domogik.package',
                          'xpltype': 'xpl-trig',
                          'command' : 'installed-packages-list'})
            Listener(self._add_to_queue_package, self.myxpl, \
                     {'schema': 'domogik.package',
                      'xpltype': 'xpl-trig'})
            Listener(self._add_to_queue_system_list, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'list'})
            Listener(self._add_to_queue_system_list, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'enable'})
            Listener(self._add_to_queue_system_list, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'disable'})
            Listener(self._add_to_queue_system_detail, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'detail'})
            Listener(self._add_to_queue_system_start, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'start'})
            Listener(self._add_to_queue_system_stop, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'stop'})
            Listener(self._add_to_queue_command, self.myxpl, \
                     {'xpltype': 'xpl-trig'})

            # Listener for hosts list
            Listener(self._list_hosts, self.myxpl, \
                     {'schema': 'hbeat.app',
                      'xpltype': 'xpl-stat'})

            # Background process to check if hosts has disappeared
            thr_hbeat = XplTimer(10, \
                                 self._refresh_status_for_list_hosts, \
                                 self.myxpl)
            thr_hbeat.start()

            self._discover_hosts()

            # Enable hbeat
            self.enable_hbeat()

            # Ask for installed packages on all hosts
            # Semaphore init for installed package list update
            self.sema_installed = Semaphore(value=1)
            self._installed_packages = {}
            if self.package_mode == True:
                self._get_installed_packages_from_manager()

            # Launch server, stats
            self.log.info("REST Initialisation OK")
            self.add_stop_cb(self.stop_http)
            self.server = None
            self.start_stats()

            self.start_http()
        except:
            self.log.error("%s" % self.get_exception())

    def _add_to_queue_package(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_package, message)

    def _add_to_queue_system_list(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_list, message)
        current_date = calendar.timegm(time.gmtime())
        self._event_dmg.add_in_queues({
            "timestamp": current_date,
            "data": "plugin-list-updated"
        })

    def _add_to_queue_system_detail(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_detail, message)

    def _add_to_queue_system_start(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_start, message)

    def _add_to_queue_system_stop(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_stop, message)

    def _add_to_queue_command(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_command, message)

    def _get_from_queue(self,
                        my_queue,
                        filter_type=None,
                        filter_schema=None,
                        filter_data=None,
                        nb_rec=0,
                        timeout=None):
        """ Encapsulation for _get_from_queue_in
            If timeout not elapsed and _get_from_queue didn't find a valid data
            call again _get_from_queue until timeout
            This encapsulation is used to process case where queue is not empty but there is
            no valid data in it and we want to wait for timeout
        """
        if timeout == None:
            timeout = self._queue_timeout
        start_time = time.time()
        while time.time() - start_time < timeout:
            try:
                return self._get_from_queue_without_waiting(
                    my_queue, filter_type, filter_schema, filter_data, nb_rec,
                    timeout)
            except Empty:
                # no data in queue for us.... let's continue until time elapsed
                # in order not rest not working so much, let it make a pause
                time.sleep(self._queue_sleep)
        # time elapsed... we can raise the Empty exception
        raise Empty

    def _get_from_queue_without_waiting(self,
                                        my_queue,
                                        filter_type=None,
                                        filter_schema=None,
                                        filter_data=None,
                                        nb_rec=0,
                                        timeout=None):
        """ Get an item from queue (recursive function)
            Checks are made on : 
            - life expectancy of message
            - filter given
            - size of queue
            If necessary, each item of queue is read.
            @param my_queue : queue to get data from
            @param filter_type : filter on a schema type
            @param filter_schema : filter on a specific schema
            @param filter_data : dictionnay of filters. Examples :
                - {"command" : "start", ...}
                - {"plugin" : "wol%", ...} : here "%" indicate that we search for something starting with "wol"
            @param nb_rec : internal parameter (do not use it for first call). Used to check recursivity VS queue size
            @param timeout : to use a different timeout from default one
        """
        if timeout == None:
            timeout = self._queue_timeout
        self.log_queue.debug("Get from queue : %s (recursivity deepth : %s)" %
                             (str(my_queue), nb_rec))
        # check if recursivity doesn't exceed queue size
        if nb_rec > my_queue.qsize():
            self.log_queue.warning(
                "Get from queue %s : number of call exceed queue size (%s) : return None"
                % (str(my_queue), my_queue.qsize()))
            # we raise an "Empty" exception because we consider that if we don't find
            # the good data, it is as if it was "empty"
            raise Empty

        msg_time, message = my_queue.get(True, timeout)

        # if message not too old, we process it
        if time.time() - msg_time < self._queue_life_expectancy:
            # no filter defined
            if filter_type == None and filter_schema == None and filter_data == None:
                self.log_queue.debug("Get from queue %s : return %s" %
                                     (str(my_queue), str(message)))
                return message

            # we want to filter data
            else:
                keep_data = True
                if filter_type != None and filter_type.lower(
                ) != message.type.lower():
                    keep_data = False
                if filter_schema != None and filter_schema.lower(
                ) != message.schema.lower():
                    keep_data = False

                if filter_data != None and keep_data == True:
                    # data
                    self.log_queue.debug("Filter on message %s WITH %s" %
                                         (message.data, filter_data))
                    for key in filter_data:
                        # take care of final "%" in order to search data starting by filter_data[key]
                        if filter_data[key][-1] == "%":
                            if message.data.has_key(key):
                                msg_data = str(message.data[key])
                                my_filter_data = str(filter_data[key])
                                len_data = len(my_filter_data) - 1
                                if msg_data[0:len_data] != my_filter_data[0:-1]:
                                    keep_data = False
                            else:
                                keep_data = False
                        # normal search
                        else:
                            if message.data.has_key(key):
                                if message.data[key].lower(
                                ) != filter_data[key].lower():
                                    keep_data = False
                            else:
                                keep_data = False

                # if message is ok for us, return it
                if keep_data == True:
                    self.log_queue.debug("Get from queue %s : return %s" %
                                         (str(my_queue), str(message)))
                    return message

                # else, message get back in queue and get another one
                else:
                    self.log_queue.debug(
                        "Get from queue %s : bad data, check another one..." %
                        (str(my_queue)))
                    self._put_in_queue(my_queue, message)
                    return self._get_from_queue_without_waiting(
                        my_queue, filter_type, filter_schema, filter_data,
                        nb_rec + 1, timeout)

        # if message too old : get an other message
        else:
            self.log_queue.debug(
                "Get from queue %s : data too old, check another one..." %
                (str(my_queue)))
            return self._get_from_queue_without_waiting(
                my_queue, filter_type, filter_schema, filter_data, nb_rec + 1,
                timeout)

    def _put_in_queue(self, my_queue, message):
        """ put a message in a named queue
            @param my_queue : queue 
            @param message : data to put in queue
        """
        self.log_queue.debug("Put in queue %s : %s" %
                             (str(my_queue), str(message)))
        try:
            my_queue.put((time.time(), message), True, self._queue_timeout)

        # Clean queue to make space
        except Full:
            msg = "Queue '%s' is full : cleaning it to make some space..." % my_queue
            self.log_queue.debug(msg)
            print(msg)
            # queue is full : start cleaning it
            nb_ck = 0
            while nb_ck < my_queue.qsize():
                (q_time, q_data) = my_queue.get()
                # data to keep
                if time.time() - self._queue_life_expectancy < q_time:
                    my_queue.put((q_time, q_data), True, self._queue_timeout)
                nb_ck += 1
            my_queue.put((time.time(), message), True, self._queue_timeout)
            self.log_queue.debug("Cleaning finished")

    def _discover_hosts(self):
        """ Send a hbeat.request to discover managers
        """
        mess = XplMessage()
        mess.set_type('xpl-cmnd')
        mess.set_target("*")
        mess.set_schema('hbeat.request')
        mess.add_data({'command': 'request'})
        self.myxpl.send(mess)

    def _list_hosts(self, message):
        """ Maintain list of Domogik hosts
            @param message : hbeat.app xpl message
        """
        tmp1 = message.source.split(".")
        tmp2 = tmp1[0].split("-")
        vendor = tmp2[0]
        device = tmp2[1]
        instance = tmp1[1]
        if vendor == "domogik" and device == "manager":
            # host not in the list
            if self._hosts_list.has_key(instance) == False:
                self._hosts_list[instance] = {"primary": False}
            self._hosts_list[instance]["status"] = "on"
            self._hosts_list[instance]["last_seen"] = time.time()
            self._hosts_list[instance]["interval"] = 60 * int(
                message.data["interval"])
            self._hosts_list[instance]["ip"] = message.data["remote-ip"]
            self._hosts_list[instance]["id"] = instance

    def _refresh_status_for_list_hosts(self):
        """ Check if hosts has disappeared
        """
        now = time.time()
        for instance in self._hosts_list:
            if (now - self._hosts_list[instance]["last_seen"] >
                    self._hosts_list[instance]["interval"]):
                self._hosts_list[instance]["status"] = "off"

    def _get_installed_packages_from_manager(self):
        """ Send a xpl message to all managers to get installed packages list
        """

        ### Send xpl message to get list
        message = XplMessage()
        message.set_type("xpl-cmnd")
        message.set_schema("domogik.package")
        message.add_data({"command": "installed-packages-list"})
        message.add_data({"host": "*"})
        self.myxpl.send(message)

    def get_installed_packages(self):
        """ return list of installed packages
            There is a semaphore in order not to return the list when it is
            updated (may be incomplete)
        """
        # developper mode : all plugins are installed

        # TODO : remove log lines after tests
        self.log.debug("*** get_installed_packages")
        self.sema_installed.acquire()
        self.log.debug("*** sema acquired")
        ret = self._installed_packages
        self.sema_installed.release()
        self.log.debug("*** sema released")
        return ret

    def _list_installed_packages(self, message):
        """ Called when a list of installed packages is received
            @param host : host
            @param pkg_type : type of package
        """
        print("Get new installed packages list")
        self.log.debug("Get new installed packages list")
        self.sema_installed.acquire()
        self.log.debug("*** sema acquired")
        self.log.debug("*** msg = %s" % message)
        # process message
        host = message.data["host"]
        self._installed_packages[host] = {}

        pkg_mgr = PackageManager()
        idx = 0
        loop_again = True
        self.log.debug("*** before while")
        while loop_again:
            try:
                self.log.debug("*** in while : idx=%s" % idx)
                pkg_type = message.data["type" + str(idx)]
                if message.data["enabled" + str(idx)].lower() == "yes":
                    enabled = True
                else:
                    enabled = False
                data = {
                    "fullname": message.data["fullname" + str(idx)],
                    "id": message.data["id" + str(idx)],
                    "version": message.data["version" + str(idx)],
                    "type": message.data["type" + str(idx)],
                    #"source" : message.data["source"+str(idx)],
                    "enabled": enabled
                }
                self.log.debug("*** call get_available_updates(%s, %s, %s)" %
                               (data["type"], data["id"], data["version"]))
                updates = pkg_mgr.get_available_updates(
                    data["type"], data["id"], data["version"])
                self.log.debug("*** after get_available_updates")
                data["updates"] = updates
                if self._installed_packages[host].has_key(pkg_type) == False:
                    self._installed_packages[host][pkg_type] = []
                self._installed_packages[host][pkg_type].append(data)
                self.log.debug("*** before idx += 1")
                idx += 1
            except KeyError:
                self.log.debug("*** except keyerror")
                loop_again = False
            except:
                self.log.debug("*** except global")
                self.log.error(
                    "Error while creating list of installed packages : %s" %
                    traceback.format_exc())
                loop_again = False
        self.log.debug("*** before release")
        self.sema_installed.release()
        self.log.debug("*** sema released")

    def start_http(self):
        """ Start HTTP Server
        """
        # Start HTTP server
        self.log.info("Start HTTP Server on %s:%s..." %
                      (self.server_ip, self.server_port))

        if self.use_ssl:
            self.server = HTTPSServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
                                         handler_params = [self])
        else:
            self.server = HTTPServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
                                         handler_params = [self])

        self.server.serve_forever()

    def stop_http(self):
        """ Stop HTTP Server
        """
        self.server.stop_handling()

    def start_stats(self):
        """ Start Statistics manager
        """
        print("Start Stats")
        self.log.info(
            "Starting statistics manager. Its logs will be in a dedicated log file"
        )
        self.stat_mgr = StatsManager(handler_params=[self], xpl=self.myxpl)
        self.stat_mgr.load()
        self.log.info("Stat manager started")

    def reload_stats(self):
        """ Reload Statistics manager
        """
        time.sleep(1)
        print("Reload Stats")
        self.log.info(
            "Reloading statistics manager. Its logs will be in a dedicated log file"
        )
        self.stat_mgr.load()

    def get_exception(self):
        """ Get exception and display it on stdout
        """
        my_exception = str(traceback.format_exc()).replace('"', "'")
        print("==== Error in REST ====")
        print(my_exception)
        print("=======================")
        return my_exception
Beispiel #4
0
class Rest(XplPlugin):
    """ REST Server 
        - create a HTTP server 
        - process REST requests
    """
        

    def __init__(self, server_ip, server_port):
        """ Initiate DbHelper, Logs and config
            Then, start HTTP server and give it initialized data
            @param server_ip :  ip of HTTP server
            @param server_port :  port of HTTP server
        """

        XplPlugin.__init__(self, name = 'rest')
        # logging initialization
        self.log.info("Rest Server initialisation...")
        self.log.debug("locale : %s %s" % locale.getdefaultlocale())

        # logging Queue activities
        log_queue = logger.Logger('rest-queues')
        self.log_queue = log_queue.get_logger('rest-queues')
        self.log_queue.info("Rest's queues activities...")
    
        # logging data manipulation initialization
        log_dm = logger.Logger('rest-dm')
        self.log_dm = log_dm.get_logger('rest-dm')
        self.log_dm.info("Rest Server Data Manipulation...")

        # API version
        self._rest_api_version = REST_API_VERSION

        # Hosts list
        self._hosts_list = {self.get_sanitized_hostname() : 
                                {"id" : self.get_sanitized_hostname(),
                                 "status" : "on",
                                 "primary" : True,
                                 "last_seen" : time.time(),
                                 "ip" : "",
                                 "interval" : "1"}}

        try:
    
            ### Config
    
            # directory data 
            cfg = Loader('domogik')
            config = cfg.load()
            conf = dict(config[1])
            self.log_dir_path = conf['log_dir_path']

            # plugin installation path
            if conf.has_key('package_path'):
                self._package_path = conf['package_path']
                self._src_prefix = None
                self.log.info("Set package path to '%s' " % self._package_path)
                print("Set package path to '%s' " % self._package_path)
                self._design_dir = "%s/domogik_packages/design/" % self._package_path
                self.package_mode = True
            else:
                self.log.info("No package path defined in config file")
                self._package_path = None
                self._src_prefix = conf['src_prefix']
                self._design_dir = "%s/share/domogik/design/" % conf['src_prefix']
                self.package_mode = False
    
            # HTTP server ip and port
            try:
                cfg_rest = Loader('rest')
                config_rest = cfg_rest.load()
                conf_rest = dict(config_rest[1])
                self.server_ip = conf_rest['rest_server_ip']
                self.server_port = conf_rest['rest_server_port']
            except KeyError:
                # default parameters
                self.server_ip = server_ip
                self.server_port = server_port
            self.log.info("Configuration : ip:port = %s:%s" % (self.server_ip, self.server_port))
    
            # SSL configuration
            try:
                cfg_rest = Loader('rest')
                config_rest = cfg_rest.load()
                conf_rest = dict(config_rest[1])
                self.use_ssl = conf_rest['rest_use_ssl']
                if self.use_ssl == "True":
                    self.use_ssl = True
                else:
                    self.use_ssl = False
                self.ssl_certificate = conf_rest['rest_ssl_certificate']
            except KeyError:
                # default parameters
                self.use_ssl = USE_SSL
                self.ssl_certificate = SSL_CERTIFICATE
            if self.use_ssl == True:
                self.log.info("Configuration : SSL support activated (certificate : %s)" % self.ssl_certificate)
            else:
                self.log.info("Configuration : SSL support not activated")
    
            # File repository
            try:
                cfg_rest = Loader('rest')
                config_rest = cfg_rest.load()
                conf_rest = dict(config_rest[1])
                self.repo_dir = conf_rest['rest_repository']
            except KeyError:
                # default parameters
                self.repo_dir = DEFAULT_REPO_DIR

            # Gloal Queues config
            self.log.debug("Get queues configuration")
            self._config = Query(self.myxpl, self.log)

            self._queue_timeout = self._config.query('rest', 'q-timeout')
            if self._queue_timeout == None:
                self._queue_timeout = QUEUE_TIMEOUT
            self._queue_timeout = float(self._queue_timeout)

            self._queue_package_size = self._config.query('rest', 'q-pkg-size')
            if self._queue_package_size == None:
                self._queue_package_size = QUEUE_PACKAGE_SIZE
            self._queue_package_size = float(self._queue_package_size)

            self._queue_size = self._config.query('rest', 'q-size')
            if self._queue_size == None:
                self._queue_size = QUEUE_SIZE
            self._queue_size = float(self._queue_size)

            self._queue_life_expectancy = self._config.query('rest', 'q-life-exp')
            if self._queue_life_expectancy == None:
                self._queue_life_expectancy = QUEUE_LIFE_EXPECTANCY
            self._queue_life_expectancy = float(self._queue_life_expectancy)

            self._queue_sleep = self._config.query('rest', 'q-sleep')
            if self._queue_sleep == None:
                self._queue_sleep = QUEUE_SLEEP
            self._queue_sleep = float(self._queue_sleep)

            # /command Queues config
            self._queue_command_size = self._config.query('rest', 'q-cmd-size')
            if self._queue_command_size == None:
                self._queue_command_size = QUEUE_COMMAND_SIZE
            self._queue_command_size = float(self._queue_command_size)

            # /event Queues config
            self._event_timeout = self._config.query('rest', 'evt-timeout')
            if self._event_timeout == None:
                self._event_timeout = EVENT_TIMEOUT
            self._event_timeout = float(self._event_timeout)

            self._queue_event_size = self._config.query('rest', 'q-evt-size')
            if self._queue_event_size == None:
                self._queue_event_size = QUEUE_EVENT_SIZE
            self._queue_event_size = float(self._queue_event_size)

            self._queue_event_timeout = self._config.query('rest', 'q-evt-timeout')
            if self._queue_event_timeout == None:
                self._queue_event_timeout = QUEUE_EVENT_TIMEOUT
            self._queue_event_timeout = float(self._queue_event_timeout)

            self._queue_event_life_expectancy = self._config.query('rest', 'q-evt-life-exp')
            if self._queue_event_life_expectancy == None:
                self._queue_event_life_expectancy = QUEUE_EVENT_LIFE_EXPECTANCY
            self._queue_event_life_expectancy = float(self._queue_event_life_expectancy)
    
            # Queues for xPL
            # Queues for packages management
            self._queue_package = Queue(self._queue_package_size)

            # Queues for domogik system actions
            self._queue_system_list = Queue(self._queue_size)
            self._queue_system_detail = Queue(self._queue_size)
            self._queue_system_start = Queue(self._queue_size)
            self._queue_system_stop = Queue(self._queue_size)

            # Queues for /command
            self._queue_command = Queue(self._queue_command_size)
    
            # Queues for /events/domogik
            self._queue_event_dmg = Queue(self._queue_event_size)
    
            # Queues for /events/request
            # this queue will be fill by stat manager
            self._event_requests = RequestEvents(self.get_stop,
                                                  self.log,
                                                  self._event_timeout,
                                                  self._queue_event_size,
                                                  self._queue_event_timeout,
                                                  self._queue_event_life_expectancy)
            self.add_stop_cb(self._event_requests.set_stop_clean)

            # Queues for /events/domogik
            # this queue will be fill by stat manager
            self._event_dmg = DmgEvents(self.get_stop,
                                     self.log,
                                     self._event_timeout,
                                     self._queue_event_size,
                                     self._queue_event_timeout,
                                     self._queue_event_life_expectancy)
            # notice : adding data in queue is made in _add_to_queue_system_list
            self.add_stop_cb(self._event_dmg.set_stop_clean)
    
            # define listeners for queues
            self.log.debug("Create listeners")
            if self.package_mode == True:
                Listener(self._list_installed_packages, self.myxpl, \
                         {'schema': 'domogik.package',
                          'xpltype': 'xpl-trig',
                          'command' : 'installed-packages-list'})
            Listener(self._add_to_queue_package, self.myxpl, \
                     {'schema': 'domogik.package',
                      'xpltype': 'xpl-trig'})
            Listener(self._add_to_queue_system_list, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'list'})
            Listener(self._add_to_queue_system_list, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'enable'})
            Listener(self._add_to_queue_system_list, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'disable'})
            Listener(self._add_to_queue_system_detail, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'detail'})
            Listener(self._add_to_queue_system_start, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'start'})
            Listener(self._add_to_queue_system_stop, self.myxpl, \
                     {'schema': 'domogik.system',
                      'xpltype': 'xpl-trig',
                      'command' : 'stop'})
            Listener(self._add_to_queue_command, self.myxpl, \
                     {'xpltype': 'xpl-trig'})

            # Listener for hosts list
            Listener(self._list_hosts, self.myxpl, \
                     {'schema': 'hbeat.app',
                      'xpltype': 'xpl-stat'})
 
            # Background process to check if hosts has disappeared
            thr_hbeat = XplTimer(10, \
                                 self._refresh_status_for_list_hosts, \
                                 self.myxpl)
            thr_hbeat.start()
   
            self._discover_hosts()
            
            # Enable hbeat
            self.enable_hbeat()

            # Ask for installed packages on all hosts
            # Semaphore init for installed package list update
            self.sema_installed = Semaphore(value=1)
            self._installed_packages = {}
            if self.package_mode == True:
                self._get_installed_packages_from_manager()

            # Launch server, stats
            self.log.info("REST Initialisation OK")
            self.add_stop_cb(self.stop_http)
            self.server = None
            self.start_stats()

            self.start_http()
        except :
            self.log.error("%s" % self.get_exception())


    def _add_to_queue_package(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_package, message)

    def _add_to_queue_system_list(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_list, message)
        current_date = calendar.timegm(time.gmtime())
        self._event_dmg.add_in_queues({"timestamp" : current_date,
                                            "data" : "plugin-list-updated"})

    def _add_to_queue_system_detail(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_detail, message)

    def _add_to_queue_system_start(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_start, message)

    def _add_to_queue_system_stop(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_system_stop, message)

    def _add_to_queue_command(self, message):
        """ Add data in a queue
        """
        self._put_in_queue(self._queue_command, message)

    def _get_from_queue(self, my_queue, filter_type = None, filter_schema = None, filter_data = None, nb_rec = 0, timeout = None):
        """ Encapsulation for _get_from_queue_in
            If timeout not elapsed and _get_from_queue didn't find a valid data
            call again _get_from_queue until timeout
            This encapsulation is used to process case where queue is not empty but there is
            no valid data in it and we want to wait for timeout
        """
        if timeout == None:
            timeout = self._queue_timeout
        start_time = time.time()
        while time.time() - start_time < timeout:
            try:
                return self._get_from_queue_without_waiting(my_queue, filter_type, filter_schema, filter_data, nb_rec, timeout)
            except Empty:
                # no data in queue for us.... let's continue until time elapsed
                # in order not rest not working so much, let it make a pause
                time.sleep(self._queue_sleep)
        # time elapsed... we can raise the Empty exception
        raise Empty



    def _get_from_queue_without_waiting(self, my_queue, filter_type = None, filter_schema = None, filter_data = None, nb_rec = 0, timeout = None):
        """ Get an item from queue (recursive function)
            Checks are made on : 
            - life expectancy of message
            - filter given
            - size of queue
            If necessary, each item of queue is read.
            @param my_queue : queue to get data from
            @param filter_type : filter on a schema type
            @param filter_schema : filter on a specific schema
            @param filter_data : dictionnay of filters. Examples :
                - {"command" : "start", ...}
                - {"plugin" : "wol%", ...} : here "%" indicate that we search for something starting with "wol"
            @param nb_rec : internal parameter (do not use it for first call). Used to check recursivity VS queue size
            @param timeout : to use a different timeout from default one
        """
        if timeout == None:
            timeout = self._queue_timeout
        self.log_queue.debug("Get from queue : %s (recursivity deepth : %s)" % (str(my_queue), nb_rec))
        # check if recursivity doesn't exceed queue size
        if nb_rec > my_queue.qsize():
            self.log_queue.warning("Get from queue %s : number of call exceed queue size (%s) : return None" % (str(my_queue), my_queue.qsize()))
            # we raise an "Empty" exception because we consider that if we don't find
            # the good data, it is as if it was "empty"
            raise Empty

        msg_time, message = my_queue.get(True, timeout)

        # if message not too old, we process it
        if time.time() - msg_time < self._queue_life_expectancy:
            # no filter defined
            if filter_type == None and filter_schema == None and filter_data == None: 
                self.log_queue.debug("Get from queue %s : return %s" % (str(my_queue), str(message)))
                return message

            # we want to filter data
            else:
                keep_data = True
                if filter_type != None and filter_type.lower() != message.type.lower():
                    keep_data = False
                if filter_schema != None and filter_schema.lower() != message.schema.lower():
                    keep_data = False

                if filter_data != None and keep_data == True:
                    # data
                    self.log_queue.debug("Filter on message %s WITH %s" % (message.data, filter_data))
                    for key in filter_data:
                        # take care of final "%" in order to search data starting by filter_data[key]
                        if filter_data[key][-1] == "%":
                            if message.data.has_key(key):
                                msg_data = str(message.data[key])
                                my_filter_data = str(filter_data[key])
                                len_data = len(my_filter_data) - 1
                                if msg_data[0:len_data] != my_filter_data[0:-1]:
                                    keep_data = False
                            else:
                                keep_data = False
                        # normal search
                        else:
                            if message.data.has_key(key):
                                if message.data[key].lower() != filter_data[key].lower():
                                    keep_data = False
                            else:
                                keep_data = False
    
                # if message is ok for us, return it
                if keep_data == True:
                    self.log_queue.debug("Get from queue %s : return %s" % (str(my_queue), str(message)))
                    return message

                # else, message get back in queue and get another one
                else:
                    self.log_queue.debug("Get from queue %s : bad data, check another one..." % (str(my_queue)))
                    self._put_in_queue(my_queue, message)
                    return self._get_from_queue_without_waiting(my_queue, filter_type, filter_schema, filter_data, nb_rec + 1, timeout)

        # if message too old : get an other message
        else:
            self.log_queue.debug("Get from queue %s : data too old, check another one..." % (str(my_queue)))
            return self._get_from_queue_without_waiting(my_queue, filter_type, filter_schema, filter_data, nb_rec + 1, timeout)

    def _put_in_queue(self, my_queue, message):
        """ put a message in a named queue
            @param my_queue : queue 
            @param message : data to put in queue
        """
        self.log_queue.debug("Put in queue %s : %s" % (str(my_queue), str(message)))
        try:
            my_queue.put((time.time(), message), True, self._queue_timeout) 

        # Clean queue to make space
        except Full:
            msg = "Queue '%s' is full : cleaning it to make some space..." % my_queue
            self.log_queue.debug(msg)
            print(msg)
            # queue is full : start cleaning it
            nb_ck = 0
            while nb_ck < my_queue.qsize():
                (q_time, q_data) = my_queue.get()
                # data to keep
                if time.time() - self._queue_life_expectancy < q_time:
                    my_queue.put((q_time, q_data), True, self._queue_timeout)
                nb_ck += 1
            my_queue.put((time.time(), message), True, self._queue_timeout) 
            self.log_queue.debug("Cleaning finished")
              
    def _discover_hosts(self):
        """ Send a hbeat.request to discover managers
        """
        mess = XplMessage()
        mess.set_type('xpl-cmnd')
        mess.set_target("*")
        mess.set_schema('hbeat.request')
        mess.add_data({'command' : 'request'})
        self.myxpl.send(mess)

    def _list_hosts(self, message):
        """ Maintain list of Domogik hosts
            @param message : hbeat.app xpl message
        """
        tmp1 = message.source.split(".")
        tmp2 = tmp1[0].split("-")
        vendor = tmp2[0]
        device = tmp2[1]
        instance = tmp1[1]
        if vendor == "domogik" and device == "manager":
             # host not in the list
             if self._hosts_list.has_key(instance) == False:
                 self._hosts_list[instance] = {"primary" : False}
             self._hosts_list[instance]["status"] = "on"
             self._hosts_list[instance]["last_seen"] = time.time()
             self._hosts_list[instance]["interval"] = 60 * int(message.data["interval"])
             self._hosts_list[instance]["ip"] = message.data["remote-ip"]
             self._hosts_list[instance]["id"] = instance
                

    def _refresh_status_for_list_hosts(self):
        """ Check if hosts has disappeared
        """
        now = time.time()
        for instance in self._hosts_list:
            if (now - self._hosts_list[instance]["last_seen"] > self._hosts_list[instance]["interval"]):
                self._hosts_list[instance]["status"] = "off"


    def _get_installed_packages_from_manager(self):
        """ Send a xpl message to all managers to get installed packages list
        """

        ### Send xpl message to get list
        message = XplMessage()
        message.set_type("xpl-cmnd")
        message.set_schema("domogik.package")
        message.add_data({"command" : "installed-packages-list"})
        message.add_data({"host" : "*"})
        self.myxpl.send(message)


    def get_installed_packages(self):
        """ return list of installed packages
            There is a semaphore in order not to return the list when it is
            updated (may be incomplete)
        """
        # developper mode : all plugins are installed

        # TODO : remove log lines after tests
        self.log.debug("*** get_installed_packages")
        self.sema_installed.acquire()
        self.log.debug("*** sema acquired")
        ret = self._installed_packages
        self.sema_installed.release()
        self.log.debug("*** sema released")
        return ret

    def _list_installed_packages(self, message):
        """ Called when a list of installed packages is received
            @param host : host
            @param pkg_type : type of package
        """
        print("Get new installed packages list")
        self.log.debug("Get new installed packages list")
        self.sema_installed.acquire()
        self.log.debug("*** sema acquired")
        self.log.debug("*** msg = %s" % message)
        # process message
        host = message.data["host"]
        self._installed_packages[host] = {}

        pkg_mgr = PackageManager()
        idx = 0
        loop_again = True
        self.log.debug("*** before while")
        while loop_again:
            try:
                self.log.debug("*** in while : idx=%s" % idx)
                pkg_type = message.data["type"+str(idx)]
                if  message.data["enabled"+str(idx)].lower() == "yes":
                    enabled = True
                else:
                    enabled = False
                data = {"fullname" : message.data["fullname"+str(idx)],
                        "id" : message.data["id"+str(idx)],
                        "version" : message.data["version"+str(idx)],
                        "type" : message.data["type"+str(idx)],
                        #"source" : message.data["source"+str(idx)],
                        "enabled" : enabled}
                self.log.debug("*** call get_available_updates(%s, %s, %s)" % (data["type"], data["id"], data["version"]))
                updates = pkg_mgr.get_available_updates(data["type"], data["id"], data["version"])
                self.log.debug("*** after get_available_updates")
                data["updates"] = updates
                if self._installed_packages[host].has_key(pkg_type) == False:
                    self._installed_packages[host][pkg_type] = []
                self._installed_packages[host][pkg_type].append(data)
                self.log.debug("*** before idx += 1")
                idx += 1
            except KeyError:
                self.log.debug("*** except keyerror")
                loop_again = False
            except:
                self.log.debug("*** except global")
                self.log.error("Error while creating list of installed packages : %s" % traceback.format_exc())
                loop_again = False
        self.log.debug("*** before release")
        self.sema_installed.release()
        self.log.debug("*** sema released")
    



    def start_http(self):
        """ Start HTTP Server
        """
        # Start HTTP server
        self.log.info("Start HTTP Server on %s:%s..." % (self.server_ip, self.server_port))

        if self.use_ssl:
            self.server = HTTPSServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
                                         handler_params = [self])
        else:
            self.server = HTTPServerWithParam((self.server_ip, int(self.server_port)), RestHandler, \
                                         handler_params = [self])

        self.server.serve_forever()



    def stop_http(self):
        """ Stop HTTP Server
        """
        self.server.stop_handling()



    def start_stats(self):
        """ Start Statistics manager
        """
        print("Start Stats")
        self.log.info("Starting statistics manager. Its logs will be in a dedicated log file")
        self.stat_mgr = StatsManager(handler_params = [self], xpl = self.myxpl)
        self.stat_mgr.load()
        self.log.info("Stat manager started")

    def reload_stats(self):
        """ Reload Statistics manager
        """
        time.sleep(1)
        print("Reload Stats")
        self.log.info("Reloading statistics manager. Its logs will be in a dedicated log file")
        self.stat_mgr.load()

    def get_exception(self):
        """ Get exception and display it on stdout
        """
        my_exception =  str(traceback.format_exc()).replace('"', "'")
        print("==== Error in REST ====")
        print(my_exception)
        print("=======================")
        return my_exception