def __init__(self, key, value, repository, autoUpdate = False, environ = None, wait = False): """Create a new ConfigSection Parameters: key The config key value The config value autoUpdate Auto update this config or not environ The environ config dict """ self._key = key self._repository = repository self._autoUpdate = autoUpdate self._environ = EnvironConfig(**environ) if environ else None self._updatedEvent = Event() self._timestamp = 0.0 self._reloadLock = None self._reloadEvent = None self._reloadedEvent = None self._reloadThread = None self._autoUpdateThread = None # Super super(ConfigSection, self).__init__() # Check if reload is required if self.ReloadRequired: self._reloadLock = RLock() self._reloadEvent = Event() self._reloadedEvent = Event() # Start reload thread thread = Thread(target = self.__reload__) thread.setDaemon(True) thread.start() self._reloadThread = thread # Update the value try: value = self.getInitialUpdatedValue() except: if self.logger.isEnabledFor(logging.DEBUG): self.logger.exception("Failed to get initial updated value, fall back to pre-defined value") self.update(value) # Check auto update (Key will empty value is not be auto updated) if key and autoUpdate: thread = Thread(target = self.__autoupdate__) thread.setDaemon(True) thread.start() self._autoUpdateThread = thread # Wait if wait: if self.ReloadRequired and self._reloadedEvent: # Wait for reloaded self._reloadedEvent.wait() elif self._updatedEvent: # Wait for updated self._updatedEvent.wait()
class ConfigSection(dict): """The config section """ logger = logging.getLogger("configmslib.section") Type = None ReloadRequired = False def __init__(self, key, value, repository, autoUpdate = False, environ = None, wait = False): """Create a new ConfigSection Parameters: key The config key value The config value autoUpdate Auto update this config or not environ The environ config dict """ self._key = key self._repository = repository self._autoUpdate = autoUpdate self._environ = EnvironConfig(**environ) if environ else None self._updatedEvent = Event() self._timestamp = 0.0 self._reloadLock = None self._reloadEvent = None self._reloadedEvent = None self._reloadThread = None self._autoUpdateThread = None # Super super(ConfigSection, self).__init__() # Check if reload is required if self.ReloadRequired: self._reloadLock = RLock() self._reloadEvent = Event() self._reloadedEvent = Event() # Start reload thread thread = Thread(target = self.__reload__) thread.setDaemon(True) thread.start() self._reloadThread = thread # Update the value try: value = self.getInitialUpdatedValue() except: if self.logger.isEnabledFor(logging.DEBUG): self.logger.exception("Failed to get initial updated value, fall back to pre-defined value") self.update(value) # Check auto update (Key will empty value is not be auto updated) if key and autoUpdate: thread = Thread(target = self.__autoupdate__) thread.setDaemon(True) thread.start() self._autoUpdateThread = thread # Wait if wait: if self.ReloadRequired and self._reloadedEvent: # Wait for reloaded self._reloadedEvent.wait() elif self._updatedEvent: # Wait for updated self._updatedEvent.wait() @property def key(self): """Get the section key """ return self._key @property def repository(self): """Get the config repository this section belongs to """ return self._repository def first(self, key, default = NoDefault): """Get first value of key """ for value in self.find(key): return value # Not found if default == NoDefault: raise KeyError(key) return default def find(self, key): """Find the values of key Returns: Yield of value """ def iterfind(obj, names): """Iterate find names in obj """ if len(names) > 0: name = names[0] # Check the obj if isinstance(obj, dict): # Find key in this dict if name in obj: # Good for v in iterfind(obj[name], names[1: ]): yield v elif isinstance(obj, (list, tuple)): # A list or tuple, iterate item for item in obj: for v in iterfind(item, names): yield v else: # Not a dict, list or tuple, stop here pass else: yield obj for value in iterfind(self, key.split(".")): yield value def __autoupdate__(self): """Auto update """ # TODO: Support modified index initialized = False self.logger.debug("[%s] Auto update thread started", self.Type) while True: # Get etcd client client = None while True: if self._repository.etcd is None: self.logger.error("[%s] Failed to watch config, no etcd client found, will retry in 30s", self.Type) time.sleep(30) continue client = self._repository.etcd break # Wait for the config # Get the read path if self._environ: path = self._environ.getEtcdPath(self._key) else: path = self._repository.environ.getEtcdPath(self._key) # Wait the config try: if not initialized: # Not initialized self.logger.debug("[%s] Watching config at path [%s]", self.Type, path) if self.update(json.loads(client.read(path).value)): initialized = True else: # Initialized, just wait self.logger.debug("[%s] Watching config at path [%s]", self.Type, path) self.update(json.loads(client.read(path, wait = True).value)) except (EtcdKeyNotFound, EtcdWatchTimedOut, EtcdEventIndexCleared): # A normal error time.sleep(10) except: # Error, wait 30s and continue watch self.logger.exception("[%s] Failed to watch etcd, will retry in 30s", self.Type) time.sleep(30) def getInitialUpdatedValue(self): """Get updated value """ if self._repository.etcd is None: raise ValueError("No etcd available") if self._environ: path = self._environ.getEtcdPath(self._key) else: path = self._repository.environ.getEtcdPath(self._key) # Get value return json.loads(self._repository.etcd.read(path).value) def update(self, value): """Update the config Parameters: value The config value Returns: Nothing """ if not isinstance(value, dict): self.logger.error("[%s] Failed to update config, value must be a dict", self.Type) return False # Validate try: self.validate(value) except: self.logger.exception("[%s] Failed to validate config value: [%s]", self.Type, json.dumps(value, ensure_ascii = False)) return False # Remove all values from self and update new values def updateConfig(): """Update the config """ self.clear() super(ConfigSection, self).update(value) self._timestamp = time.time() # Update if self._reloadLock: with self._reloadLock: updateConfig() else: updateConfig() # If reload is required if self.ReloadRequired: self._reloadEvent.set() # Updated self._updatedEvent.set() if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug("[%s] Config updated [%s]", self.Type, json.dumps(value, ensure_ascii = False)) # Done return True def validate(self, value): """Validate the config value """ pass def __reload__(self): """Reload this config """ self.logger.debug("[%s] Reload thread started", self.Type) reloadedTimestamp = 0.0 while True: if reloadedTimestamp >= self._timestamp: # Wait for reload event self._reloadEvent.wait() self._reloadEvent.clear() # Start reload until reload succeed while True: try: # Lock the reload lock, copy timestamp and config with self._reloadLock: reloadedTimestamp = self._timestamp config = dict(self) # Run reload self.reload(config) except: self.logger.exception("[%s] Failed to reload config, will retry in 30s", self.Type) time.sleep(30) else: self._reloadedEvent.set() break def reload(self, config): """Reload this config """ pass def close(self): """Close the config """ pass
def main(): """The main entry """ args = getArguments() # Connect etcd, will try in the following order: # - config # - host # - service domain # - system search domain as service domain environ = None try: if args.config: logging.info("Try to connect to etcd by config [%s]", args.config) etcdClient, environ = getEtcdClientByConfig(args.config) elif args.host: logging.info("Try to connect to etcd by host [%s] port [%d]", args.host, args.port) etcdClient = etcd.Client(host = args.host, port = args.port, allow_reconnect = True, protocol = args.scheme) elif args.srvDomain: logging.info("Try to connect to etcd by service domain [%s]", args.srvDomain) etcdClient = etcd.Client(srv_domain = args.srvDomain, allow_reconnect = True, protocol = args.scheme) else: logging.info("Try to connect to etcd by system search domain") etcdClient = getEtcdClientBySystemSearchDomain(args.scheme) except Exception as error: logging.error("Failed to connect to etcd, error: [%s]", error) return 1 # Get environ if not environ: environ = EnvironConfig() if args.repository: environ["repository"] = args.repository if args.environ: environ["name"] = args.environ logging.info("Set environ [%s] --> [%s]", environ.repository, environ.name) try: if args.action == "set": # Set the config for kv in args.kvs: idx = kv.find("=") if idx == -1: logging.error("Bad kv value: [%s]", kv) return 1 key, value = kv[: idx], kv[idx + 1: ] if value.startswith("@"): # From file logging.info("Load value of key [%s] from file [%s]", key, value) value = loadValueFile(value[1: ]) print "Set key", termcolor.colored(key, "yellow") etcdClient.write(environ.getEtcdPath(key), value) elif args.action == "get": # Get the config for key in args.keys: result = etcdClient.read(environ.getEtcdPath(key)) print termcolor.colored(key, "yellow"), result.value else: logging.error("Unknown action [%s]", args.action) return 1 except etcd.EtcdKeyNotFound: # Config not found print termcolor.colored("Config not found", "red") # Done return 0