Ejemplo n.º 1
0
 def __init__(self, key, value, repository, autoUpdate = False, environ = None, wait = False):
     """Create a new ConfigSection
     Parameters:
         key                             The config key
         value                           The config value
         autoUpdate                      Auto update this config or not
         environ                         The environ config dict
     """
     self._key = key
     self._repository = repository
     self._autoUpdate = autoUpdate
     self._environ = EnvironConfig(**environ) if environ else None
     self._updatedEvent = Event()
     self._timestamp = 0.0
     self._reloadLock = None
     self._reloadEvent = None
     self._reloadedEvent = None
     self._reloadThread = None
     self._autoUpdateThread = None
     # Super
     super(ConfigSection, self).__init__()
     # Check if reload is required
     if self.ReloadRequired:
         self._reloadLock = RLock()
         self._reloadEvent = Event()
         self._reloadedEvent = Event()
         # Start reload thread
         thread = Thread(target = self.__reload__)
         thread.setDaemon(True)
         thread.start()
         self._reloadThread = thread
     # Update the value
     try:
         value = self.getInitialUpdatedValue()
     except:
         if self.logger.isEnabledFor(logging.DEBUG):
             self.logger.exception("Failed to get initial updated value, fall back to pre-defined value")
     self.update(value)
     # Check auto update (Key will empty value is not be auto updated)
     if key and autoUpdate:
         thread = Thread(target = self.__autoupdate__)
         thread.setDaemon(True)
         thread.start()
         self._autoUpdateThread = thread
     # Wait
     if wait:
         if self.ReloadRequired and self._reloadedEvent:
             # Wait for reloaded
             self._reloadedEvent.wait()
         elif self._updatedEvent:
             # Wait for updated
             self._updatedEvent.wait()
Ejemplo n.º 2
0
class ConfigSection(dict):
    """The config section
    """
    logger = logging.getLogger("configmslib.section")

    Type = None
    ReloadRequired = False

    def __init__(self, key, value, repository, autoUpdate = False, environ = None, wait = False):
        """Create a new ConfigSection
        Parameters:
            key                             The config key
            value                           The config value
            autoUpdate                      Auto update this config or not
            environ                         The environ config dict
        """
        self._key = key
        self._repository = repository
        self._autoUpdate = autoUpdate
        self._environ = EnvironConfig(**environ) if environ else None
        self._updatedEvent = Event()
        self._timestamp = 0.0
        self._reloadLock = None
        self._reloadEvent = None
        self._reloadedEvent = None
        self._reloadThread = None
        self._autoUpdateThread = None
        # Super
        super(ConfigSection, self).__init__()
        # Check if reload is required
        if self.ReloadRequired:
            self._reloadLock = RLock()
            self._reloadEvent = Event()
            self._reloadedEvent = Event()
            # Start reload thread
            thread = Thread(target = self.__reload__)
            thread.setDaemon(True)
            thread.start()
            self._reloadThread = thread
        # Update the value
        try:
            value = self.getInitialUpdatedValue()
        except:
            if self.logger.isEnabledFor(logging.DEBUG):
                self.logger.exception("Failed to get initial updated value, fall back to pre-defined value")
        self.update(value)
        # Check auto update (Key will empty value is not be auto updated)
        if key and autoUpdate:
            thread = Thread(target = self.__autoupdate__)
            thread.setDaemon(True)
            thread.start()
            self._autoUpdateThread = thread
        # Wait
        if wait:
            if self.ReloadRequired and self._reloadedEvent:
                # Wait for reloaded
                self._reloadedEvent.wait()
            elif self._updatedEvent:
                # Wait for updated
                self._updatedEvent.wait()

    @property
    def key(self):
        """Get the section key
        """
        return self._key

    @property
    def repository(self):
        """Get the config repository this section belongs to
        """
        return self._repository

    def first(self, key, default = NoDefault):
        """Get first value of key
        """
        for value in self.find(key):
            return value
        # Not found
        if default == NoDefault:
            raise KeyError(key)
        return default

    def find(self, key):
        """Find the values of key
        Returns:
            Yield of value
        """
        def iterfind(obj, names):
            """Iterate find names in obj
            """
            if len(names) > 0:
                name = names[0]
                # Check the obj
                if isinstance(obj, dict):
                    # Find key in this dict
                    if name in obj:
                        # Good
                        for v in iterfind(obj[name], names[1: ]):
                            yield v
                elif isinstance(obj, (list, tuple)):
                    # A list or tuple, iterate item
                    for item in obj:
                        for v in iterfind(item, names):
                            yield v
                else:
                    # Not a dict, list or tuple, stop here
                    pass
            else:
                yield obj
        for value in iterfind(self, key.split(".")):
            yield value

    def __autoupdate__(self):
        """Auto update
        """
        # TODO: Support modified index
        initialized = False
        self.logger.debug("[%s] Auto update thread started", self.Type)
        while True:
            # Get etcd client
            client = None
            while True:
                if self._repository.etcd is None:
                    self.logger.error("[%s] Failed to watch config, no etcd client found, will retry in 30s", self.Type)
                    time.sleep(30)
                    continue
                client = self._repository.etcd
                break
            # Wait for the config
            # Get the read path
            if self._environ:
                path = self._environ.getEtcdPath(self._key)
            else:
                path = self._repository.environ.getEtcdPath(self._key)
            # Wait the config
            try:
                if not initialized:
                    # Not initialized
                    self.logger.debug("[%s] Watching config at path [%s]", self.Type, path)
                    if self.update(json.loads(client.read(path).value)):
                        initialized = True
                else:
                    # Initialized, just wait
                    self.logger.debug("[%s] Watching config at path [%s]", self.Type, path)
                    self.update(json.loads(client.read(path, wait = True).value))
            except (EtcdKeyNotFound, EtcdWatchTimedOut, EtcdEventIndexCleared):
                # A normal error
                time.sleep(10)
            except:
                # Error, wait 30s and continue watch
                self.logger.exception("[%s] Failed to watch etcd, will retry in 30s", self.Type)
                time.sleep(30)

    def getInitialUpdatedValue(self):
        """Get updated value
        """
        if self._repository.etcd is None:
            raise ValueError("No etcd available")
        if self._environ:
            path = self._environ.getEtcdPath(self._key)
        else:
            path = self._repository.environ.getEtcdPath(self._key)
        # Get value
        return json.loads(self._repository.etcd.read(path).value)

    def update(self, value):
        """Update the config
        Parameters:
            value                           The config value
        Returns:
            Nothing
        """
        if not isinstance(value, dict):
            self.logger.error("[%s] Failed to update config, value must be a dict", self.Type)
            return False
        # Validate
        try:
            self.validate(value)
        except:
            self.logger.exception("[%s] Failed to validate config value: [%s]", self.Type, json.dumps(value, ensure_ascii = False))
            return False
        # Remove all values from self and update new values
        def updateConfig():
            """Update the config
            """
            self.clear()
            super(ConfigSection, self).update(value)
            self._timestamp = time.time()
        # Update
        if self._reloadLock:
            with self._reloadLock:
                updateConfig()
        else:
            updateConfig()
        # If reload is required
        if self.ReloadRequired:
            self._reloadEvent.set()
        # Updated
        self._updatedEvent.set()
        if self.logger.isEnabledFor(logging.DEBUG):
            self.logger.debug("[%s] Config updated [%s]", self.Type, json.dumps(value, ensure_ascii = False))
        # Done
        return True

    def validate(self, value):
        """Validate the config value
        """
        pass

    def __reload__(self):
        """Reload this config
        """
        self.logger.debug("[%s] Reload thread started", self.Type)
        reloadedTimestamp = 0.0
        while True:
            if reloadedTimestamp >= self._timestamp:
                # Wait for reload event
                self._reloadEvent.wait()
                self._reloadEvent.clear()
            # Start reload until reload succeed
            while True:
                try:
                    # Lock the reload lock, copy timestamp and config
                    with self._reloadLock:
                        reloadedTimestamp = self._timestamp
                        config = dict(self)
                    # Run reload
                    self.reload(config)
                except:
                    self.logger.exception("[%s] Failed to reload config, will retry in 30s", self.Type)
                    time.sleep(30)
                else:
                    self._reloadedEvent.set()
                    break

    def reload(self, config):
        """Reload this config
        """
        pass

    def close(self):
        """Close the config
        """
        pass
Ejemplo n.º 3
0
def main():
    """The main entry
    """
    args = getArguments()
    # Connect etcd, will try in the following order:
    #   - config
    #   - host
    #   - service domain
    #   - system search domain as service domain
    environ = None
    try:
        if args.config:
            logging.info("Try to connect to etcd by config [%s]", args.config)
            etcdClient, environ = getEtcdClientByConfig(args.config)
        elif args.host:
            logging.info("Try to connect to etcd by host [%s] port [%d]", args.host, args.port)
            etcdClient = etcd.Client(host = args.host, port = args.port, allow_reconnect = True, protocol = args.scheme)
        elif args.srvDomain:
            logging.info("Try to connect to etcd by service domain [%s]", args.srvDomain)
            etcdClient = etcd.Client(srv_domain = args.srvDomain, allow_reconnect = True, protocol = args.scheme)
        else:
            logging.info("Try to connect to etcd by system search domain")
            etcdClient = getEtcdClientBySystemSearchDomain(args.scheme)
    except Exception as error:
        logging.error("Failed to connect to etcd, error: [%s]", error)
        return 1
    # Get environ
    if not environ:
        environ = EnvironConfig()
    if args.repository:
        environ["repository"] = args.repository
    if args.environ:
        environ["name"] = args.environ
    logging.info("Set environ [%s] --> [%s]", environ.repository, environ.name)
    try:
        if args.action == "set":
            # Set the config
            for kv in args.kvs:
                idx = kv.find("=")
                if idx == -1:
                    logging.error("Bad kv value: [%s]", kv)
                    return 1
                key, value = kv[: idx], kv[idx + 1: ]
                if value.startswith("@"):
                    # From file
                    logging.info("Load value of key [%s] from file [%s]", key, value)
                    value = loadValueFile(value[1: ])
                print "Set key", termcolor.colored(key, "yellow")
                etcdClient.write(environ.getEtcdPath(key), value)
        elif args.action == "get":
            # Get the config
            for key in args.keys:
                result = etcdClient.read(environ.getEtcdPath(key))
                print termcolor.colored(key, "yellow"), result.value
        else:
            logging.error("Unknown action [%s]", args.action)
            return 1
    except etcd.EtcdKeyNotFound:
        # Config not found
        print termcolor.colored("Config not found", "red")
    # Done
    return 0