def __init__(self, key, value, repository, autoUpdate = False, environ = None, wait = False):
"""Create a new ConfigSection
Parameters:
key The config key
value The config value
autoUpdate Auto update this config or not
environ The environ config dict
"""
self._key = key
self._repository = repository
self._autoUpdate = autoUpdate
self._environ = EnvironConfig(**environ) if environ else None
self._updatedEvent = Event()
self._timestamp = 0.0
self._reloadLock = None
self._reloadEvent = None
self._reloadedEvent = None
self._reloadThread = None
self._autoUpdateThread = None
# Super
super(ConfigSection, self).__init__()
# Check if reload is required
if self.ReloadRequired:
self._reloadLock = RLock()
self._reloadEvent = Event()
self._reloadedEvent = Event()
# Start reload thread
thread = Thread(target = self.__reload__)
thread.setDaemon(True)
thread.start()
self._reloadThread = thread
# Update the value
try:
value = self.getInitialUpdatedValue()
except:
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.exception("Failed to get initial updated value, fall back to pre-defined value")
self.update(value)
# Check auto update (Key will empty value is not be auto updated)
if key and autoUpdate:
thread = Thread(target = self.__autoupdate__)
thread.setDaemon(True)
thread.start()
self._autoUpdateThread = thread
# Wait
if wait:
if self.ReloadRequired and self._reloadedEvent:
# Wait for reloaded
self._reloadedEvent.wait()
elif self._updatedEvent:
# Wait for updated
self._updatedEvent.wait()
class ConfigSection(dict):
"""The config section
"""
logger = logging.getLogger("configmslib.section")
Type = None
ReloadRequired = False
def __init__(self, key, value, repository, autoUpdate = False, environ = None, wait = False):
"""Create a new ConfigSection
Parameters:
key The config key
value The config value
autoUpdate Auto update this config or not
environ The environ config dict
"""
self._key = key
self._repository = repository
self._autoUpdate = autoUpdate
self._environ = EnvironConfig(**environ) if environ else None
self._updatedEvent = Event()
self._timestamp = 0.0
self._reloadLock = None
self._reloadEvent = None
self._reloadedEvent = None
self._reloadThread = None
self._autoUpdateThread = None
# Super
super(ConfigSection, self).__init__()
# Check if reload is required
if self.ReloadRequired:
self._reloadLock = RLock()
self._reloadEvent = Event()
self._reloadedEvent = Event()
# Start reload thread
thread = Thread(target = self.__reload__)
thread.setDaemon(True)
thread.start()
self._reloadThread = thread
# Update the value
try:
value = self.getInitialUpdatedValue()
except:
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.exception("Failed to get initial updated value, fall back to pre-defined value")
self.update(value)
# Check auto update (Key will empty value is not be auto updated)
if key and autoUpdate:
thread = Thread(target = self.__autoupdate__)
thread.setDaemon(True)
thread.start()
self._autoUpdateThread = thread
# Wait
if wait:
if self.ReloadRequired and self._reloadedEvent:
# Wait for reloaded
self._reloadedEvent.wait()
elif self._updatedEvent:
# Wait for updated
self._updatedEvent.wait()
@property
def key(self):
"""Get the section key
"""
return self._key
@property
def repository(self):
"""Get the config repository this section belongs to
"""
return self._repository
def first(self, key, default = NoDefault):
"""Get first value of key
"""
for value in self.find(key):
return value
# Not found
if default == NoDefault:
raise KeyError(key)
return default
def find(self, key):
"""Find the values of key
Returns:
Yield of value
"""
def iterfind(obj, names):
"""Iterate find names in obj
"""
if len(names) > 0:
name = names[0]
# Check the obj
if isinstance(obj, dict):
# Find key in this dict
if name in obj:
# Good
for v in iterfind(obj[name], names[1: ]):
yield v
elif isinstance(obj, (list, tuple)):
# A list or tuple, iterate item
for item in obj:
for v in iterfind(item, names):
yield v
else:
# Not a dict, list or tuple, stop here
pass
else:
yield obj
for value in iterfind(self, key.split(".")):
yield value
def __autoupdate__(self):
"""Auto update
"""
# TODO: Support modified index
initialized = False
self.logger.debug("[%s] Auto update thread started", self.Type)
while True:
# Get etcd client
client = None
while True:
if self._repository.etcd is None:
self.logger.error("[%s] Failed to watch config, no etcd client found, will retry in 30s", self.Type)
time.sleep(30)
continue
client = self._repository.etcd
break
# Wait for the config
# Get the read path
if self._environ:
path = self._environ.getEtcdPath(self._key)
else:
path = self._repository.environ.getEtcdPath(self._key)
# Wait the config
try:
if not initialized:
# Not initialized
self.logger.debug("[%s] Watching config at path [%s]", self.Type, path)
if self.update(json.loads(client.read(path).value)):
initialized = True
else:
# Initialized, just wait
self.logger.debug("[%s] Watching config at path [%s]", self.Type, path)
self.update(json.loads(client.read(path, wait = True).value))
except (EtcdKeyNotFound, EtcdWatchTimedOut, EtcdEventIndexCleared):
# A normal error
time.sleep(10)
except:
# Error, wait 30s and continue watch
self.logger.exception("[%s] Failed to watch etcd, will retry in 30s", self.Type)
time.sleep(30)
def getInitialUpdatedValue(self):
"""Get updated value
"""
if self._repository.etcd is None:
raise ValueError("No etcd available")
if self._environ:
path = self._environ.getEtcdPath(self._key)
else:
path = self._repository.environ.getEtcdPath(self._key)
# Get value
return json.loads(self._repository.etcd.read(path).value)
def update(self, value):
"""Update the config
Parameters:
value The config value
Returns:
Nothing
"""
if not isinstance(value, dict):
self.logger.error("[%s] Failed to update config, value must be a dict", self.Type)
return False
# Validate
try:
self.validate(value)
except:
self.logger.exception("[%s] Failed to validate config value: [%s]", self.Type, json.dumps(value, ensure_ascii = False))
return False
# Remove all values from self and update new values
def updateConfig():
"""Update the config
"""
self.clear()
super(ConfigSection, self).update(value)
self._timestamp = time.time()
# Update
if self._reloadLock:
with self._reloadLock:
updateConfig()
else:
updateConfig()
# If reload is required
if self.ReloadRequired:
self._reloadEvent.set()
# Updated
self._updatedEvent.set()
if self.logger.isEnabledFor(logging.DEBUG):
self.logger.debug("[%s] Config updated [%s]", self.Type, json.dumps(value, ensure_ascii = False))
# Done
return True
def validate(self, value):
"""Validate the config value
"""
pass
def __reload__(self):
"""Reload this config
"""
self.logger.debug("[%s] Reload thread started", self.Type)
reloadedTimestamp = 0.0
while True:
if reloadedTimestamp >= self._timestamp:
# Wait for reload event
self._reloadEvent.wait()
self._reloadEvent.clear()
# Start reload until reload succeed
while True:
try:
# Lock the reload lock, copy timestamp and config
with self._reloadLock:
reloadedTimestamp = self._timestamp
config = dict(self)
# Run reload
self.reload(config)
except:
self.logger.exception("[%s] Failed to reload config, will retry in 30s", self.Type)
time.sleep(30)
else:
self._reloadedEvent.set()
break
def reload(self, config):
"""Reload this config
"""
pass
def close(self):
"""Close the config
"""
pass
def main():
"""The main entry
"""
args = getArguments()
# Connect etcd, will try in the following order:
# - config
# - host
# - service domain
# - system search domain as service domain
environ = None
try:
if args.config:
logging.info("Try to connect to etcd by config [%s]", args.config)
etcdClient, environ = getEtcdClientByConfig(args.config)
elif args.host:
logging.info("Try to connect to etcd by host [%s] port [%d]", args.host, args.port)
etcdClient = etcd.Client(host = args.host, port = args.port, allow_reconnect = True, protocol = args.scheme)
elif args.srvDomain:
logging.info("Try to connect to etcd by service domain [%s]", args.srvDomain)
etcdClient = etcd.Client(srv_domain = args.srvDomain, allow_reconnect = True, protocol = args.scheme)
else:
logging.info("Try to connect to etcd by system search domain")
etcdClient = getEtcdClientBySystemSearchDomain(args.scheme)
except Exception as error:
logging.error("Failed to connect to etcd, error: [%s]", error)
return 1
# Get environ
if not environ:
environ = EnvironConfig()
if args.repository:
environ["repository"] = args.repository
if args.environ:
environ["name"] = args.environ
logging.info("Set environ [%s] --> [%s]", environ.repository, environ.name)
try:
if args.action == "set":
# Set the config
for kv in args.kvs:
idx = kv.find("=")
if idx == -1:
logging.error("Bad kv value: [%s]", kv)
return 1
key, value = kv[: idx], kv[idx + 1: ]
if value.startswith("@"):
# From file
logging.info("Load value of key [%s] from file [%s]", key, value)
value = loadValueFile(value[1: ])
print "Set key", termcolor.colored(key, "yellow")
etcdClient.write(environ.getEtcdPath(key), value)
elif args.action == "get":
# Get the config
for key in args.keys:
result = etcdClient.read(environ.getEtcdPath(key))
print termcolor.colored(key, "yellow"), result.value
else:
logging.error("Unknown action [%s]", args.action)
return 1
except etcd.EtcdKeyNotFound:
# Config not found
print termcolor.colored("Config not found", "red")
# Done
return 0