class Watcher(threading.Thread):
def __init__(self, confman_etcd_root):
threading.Thread.__init__(self)
logs = Logger(filename = "watcher", \
logger_name = "Config Watcher", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
normalized_confman_etcd_root = os.path.normpath(confman_etcd_root)
confman_config = f"{normalized_confman_etcd_root}/occonfman"
self.confman_key = {}
self.conf = []
try:
self.conf = self.etcd.get_key(confman_config)
print("Getting main config for occonfman")
logs.info("Getting main config for occonfman")
if (self.conf[0]):
self.confman_key = json.loads(self.conf[0].decode("utf-8"))
logs.info(f"Successfully retrieved main config for occonfman")
else:
logs.error(f"Confman config may be empty {self.conf}")
except Exception as e:
print(
f"Watcher was unable to retrieve its main config file {confman_config}"
)
logs.log.exception(
f"Watcher was unable to get its config file {confman_config}")
raise (e)
# At this point we know that ectd is available
# if self.confman_key is not present or it is empty, add default config to ectd
if not self.confman_key:
self.confman_key = {
"config_key_prefix":
f"{normalized_confman_etcd_root}/configs/",
"status_key_prefix": f"{normalized_confman_etcd_root}/status/",
"appconf_json_key":
f"{normalized_confman_etcd_root}/occonfman/",
"appconf_json_dir": "/aux0/customer/occonfman/"
}
logs.warning(
f"{confman_config} is not present in etcd or empty. Using default: {self.confman_key}"
)
self.etcd.write(confman_config, json.dumps(self.confman_key))
"""
Initialize the configurations
- read all application json config files and copy them to etcd
- check if config key exists, if not, copy its content and commands
- run through all configs and process them one by one
"""
logs.info("Starting config initialization procedure.")
if (self.confman_key):
self.initial = InitializeConfig(
self.confman_key["config_key_prefix"],
self.confman_key["status_key_prefix"])
logs.info("Applying preset configs for initialization:")
self.initial.apply_preset_configs(
self.confman_key["appconf_json_dir"],
self.confman_key["appconf_json_key"], 0)
logs.info(
"Going through each config file in etcd and processing its content and commands:"
)
self.initial.preconfigure()
else:
logs.error(
f"No initialization will be done, since main confman key was not present: {confman_config}"
)
print(
f"No initialization, because there's no confman key: {confman_config}"
)
logs.clear_handler()
def watcher(self, events):
logs = Logger(filename = "watcher", \
logger_name = "Config Watcher", \
dirname="/aux1/occonfman/")
for event in events.events:
logs.info(f"New event received: {event}")
# print(event.mod_revision)
new_event = Worker(event, self.confman_key["status_key_prefix"])
new_event.start()
logs.clear_handler()
def run(self):
self.etcd.add_watch_prefix_callback(
self.confman_key["config_key_prefix"], self.watcher)
class InitializeConfig():
def __init__(self, key_prefix, status_path):
self.hostname = os.uname()[1]
logs = Logger(filename = "initialize_config", \
logger_name = "Config Initialize", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
self.key_pref = key_prefix
self.status_path = status_path
logs.clear_handler()
def check_affected(self, val):
command_list = []
logs = Logger(filename = "occonfman", \
logger_name = "Config Initialize check_affected", \
dirname="/aux1/occonfman/")
try:
json_val = json.loads(val)
except:
logs.error(f"Invalid json: {val} ")
return []
if self.hostname in json_val["commands"].keys():
command_list += json_val["commands"][self.hostname]
logs.clear_handler()
return command_list
def apply_preset_configs(self, json_path, etcd_path, reset):
logs = Logger(filename = "occonfman", \
logger_name = "Config Initialize apply_preset_configs", \
dirname="/aux1/occonfman/")
app_path = Path(json_path)
json_data = []
if app_path.is_dir():
for confs in app_path.glob('*.json'):
if confs.is_file():
with open(confs, "r") as conf:
try:
conf_content = conf.read()
json_data += json.loads(conf_content)
etcd_fullpath = etcd_path.rstrip('/') + '/' + str(
confs.name)
self.etcd.write(etcd_fullpath, conf_content)
except:
print(f"Could not load {confs} file")
logs.error(f"Cound not load {confs} file")
continue
else:
print(f"Filepath does not exist: {json_path}")
logs.error(f"Filepath does not exist: {json_path}")
configdict = {}
for cnt in json_data:
if cnt["config"] in configdict.keys():
for arrv in cnt["command"]:
configdict[cnt["config"]].append(arrv)
else:
configdict[cnt["config"]] = list()
for arrv in cnt["command"]:
configdict[cnt["config"]].append(arrv)
for config_data in json_data:
config_key_path = self.key_pref.rstrip('/') + config_data["config"]
config_key = self.etcd.get_key(config_key_path)
if (config_key[0]):
try:
json_content = json.loads(config_key[0].decode("utf-8"))
except:
print("Invalid json for: {} \n Content: {}".format(
config_key_path, config_key[0].decode("utf-8")))
logs.error(
f"Invalid json for: {config_key_path} \n Content: {config_key[0].decode('utf-8')} "
)
continue
if self.hostname in json_content["commands"].keys(
) and not reset:
print(
"Hostname already exists in commands and reset is not set, so no overwrite"
)
logs.info(
"Hostname already exists in commands and reset is not set, so no overwrite"
)
else:
json_content["commands"][self.hostname] = configdict[
config_data["config"]]
self.etcd.write(config_key_path, json_content)
print(
f"Key exists, but {self.hostname} is not in commands or reset is given"
)
logs.info(
f"Key exists, but {self.hostname} is not in commands or reset is given"
)
else:
print(
"Found config file without a key in etcd. Attempting to generate it from template."
)
logs.error(
"Found config file without a key in etcd. Attempting to generate it from template."
)
if Path(config_data["config"]).is_file():
fcontent = ''
json_content = {}
with open(config_data["config"], "r") as content_file:
fcontent = content_file.read()
json_content["content"] = fcontent
json_content["commands"] = dict()
json_content["commands"][self.hostname] = configdict[
config_data["config"]]
json_content["path"] = config_data["config"]
self.etcd.write(config_key_path, json.dumps(json_content))
logs.clear_handler()
def preconfigure(self):
logs = Logger(filename = "occonfman", \
logger_name = "Config Initializa - preconfigure", \
dirname="/aux1/occonfman/")
commands_set = set()
for kvmeta in self.etcd.get_prefix_real(self.key_pref):
ckey = kvmeta[1].key.decode('utf-8')
cval = kvmeta[0].decode("utf-8")
affected_commands = self.check_affected(cval)
if not affected_commands:
print(f"{ckey} does not concern my hostname {self.hostname}")
logs.info(
f"{ckey} does not concern my hostname {self.hostname}")
continue
process_config = ProcessConfig(kvmeta, self.status_path, 1)
process_config.process_config()
# for single_command in affected_commands:
# commands_set.add(single_command)
# for command in commands_set:
# subprocess.run(str(command), shell = True)
logs.clear_handler()