def delete(self):
parser = reqparse.RequestParser()
data = request.get_json()
resp_list = []
etcd_manager = EtcdManagement()
deleted_flag = None
for key_path in data:
try:
deleted_flag = etcd_manager.remove_key(key_path)
except:
resp_list.append({
"key": key_path,
"status": "failed",
"code": 1002
})
continue
if deleted_flag:
resp_list.append({"key": key_path, "status": "success"})
else:
resp_list.append({
"key": key_path,
"status": "failed",
"code": 1002
})
deleted_flag = None
return jsonify(resp_list)
def post(self):
mod_revision = None
parser = reqparse.RequestParser()
parser.add_argument('key', required=True)
parser.add_argument('value', required=True)
args = parser.parse_args()
logger = Logger(filename = "kvs_wrapper", \
logger_name = "Etcd_kvs post", \
dirname="/aux1/ockvsman/logs/")
logger.info("Handle params from web.")
etcd_manager = EtcdManagement()
try:
etcd_manager.write(new_key=args['key'], value=args['value'])
mod_revision = etcd_manager.get_mod_revision(args['key'])
except:
logger.info("Request can't be executed, Error code => 1000")
return {
'message': "Request can't be registered",
"code": 1000
}, 500
logger.info(
"Key was registered successfully, mod_revision => {}".format(
mod_revision))
logger.clear_handler()
return {
'message': 'Key was registered successfully',
'mod_revision': mod_revision
}, 200
def post(self):
parser = reqparse.RequestParser()
data = request.get_json()
etcdman = EtcdManagement()
resp_list = []
for dict_obj in data:
try:
etcdman.write(new_key=dict_obj["key"], value=dict_obj["value"])
except:
resp_list.append({
"key": dict_obj["key"],
"status": "failed",
"code": 1000
})
continue
resp_list.append({
"key":
dict_obj["key"],
"status":
"success",
"mod_revision":
etcdman.get_mod_revision(dict_obj['key'])
})
return jsonify(resp_list)
def __init__(self):
"""
Constructor
Args:
available_nodes(list)
"""
###etcd way
self.etcd_manager = EtcdManagement()
def __init__(self, key_prefix, status_path):
self.hostname = os.uname()[1]
logs = Logger(filename = "initialize_config", \
logger_name = "Config Initialize", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
self.key_pref = key_prefix
self.status_path = status_path
logs.clear_handler()
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('prefix', type=str)
args = parser.parse_args()
etcd_manager = EtcdManagement()
key_data = None
try:
key_data = etcd_manager.read_key_prefix(args['prefix'])
return jsonify(key_data)
except:
return {'message': "Keys can't be get!!", "code": 1001}, 404
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('etcd_key', required=True)
args = parser.parse_args()
etcd_manager = EtcdManagement()
key_data = None
try:
key_data = etcd_manager.read_key(args['etcd_key'])
return {
"value": key_data,
"mod_revision": etcd_manager.get_mod_revision(args['etcd_key'])
}, 200
except:
return {'message': "Key can't be get!!", "code": 1001}, 404
def get_initial_obj_instances():
"""
Returns:
tuple
"""
return (DecisionMaker(), \
EtcdManagement(), \
ConfigSupervisor(),\
ContainerManagement())
def delete(self):
parser = reqparse.RequestParser()
parser.add_argument('key', type=str)
args = parser.parse_args()
etcd_manager = EtcdManagement()
deleted_flag = None
try:
deleted_flag = etcd_manager.remove_key(args["key"])
except:
return {'message': "Request can't be executed", "code": 1000}, 500
if deleted_flag:
return {'message': 'Key was deleted successfully'}, 200
else:
return {
'message': "Key wasn't deleted successfully",
"code": 1002
}, 409
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('key', required=True)
parser.add_argument('value', required=True)
args = parser.parse_args()
etcd_manager = EtcdManagement()
try:
etcd_manager.write(new_key=args['key'], value=args['value'])
mod_revision = etcd_manager.get_mod_revision(args['key'])
return {
'message': 'Key was registered successfully',
"mod_revision": mod_revision
}, 200
except:
return {'message': "Request can't be executed", "code": 1000}, 500
def __init__(self, event, status_path, initial):
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
self.initial = initial
if (self.initial):
self.event_value = json.loads(event[0].decode("utf-8"))
self.event_revision = str(
self.etcd.get_mod_revision(event[1].key.decode("utf-8")))
print(self.event_revision)
log.info(f"Event revision is: {self.event_revision}")
else:
self.event_value = json.loads(event.value)
self.event_revision = event.mod_revision
self.hostname = os.uname()[1]
self.status_path = status_path
log.clear_handler()
def __init__(self):
"""
Constructor of swarm manager
Args:
available_nodes(list)
platform_nodes(list)
user(str)
password(str)
master_node(str)
token(str)
"""
###orchastrator.json way
# self.ssh_client = paramiko.SSHClient()
# self.ssh_client.load_system_host_keys()
# self.available_nodes = parse_config("orchastrator.json")["available_nodes"]
# self.platform_nodes = parse_config("orchastrator.json")["platform_nodes"]
# self.user = parse_config("orchastrator.json")["user"]
# self.password = parse_config("orchastrator.json")["password"]
# self.master_nodes = parse_config("orchastrator.json")["master_nodes"]
# self.__master = parse_config("orchastrator.json")["master"]
# self.__token = parse_config("orchastrator.json")["token"]
###orchastrator.json way
###etcd way
self.etcd_manager = EtcdManagement()
self.orchastrator_config = self.etcd_manager.get_etcd_orchestrator_config()['platform']['orchestrator']
self.ssh_client = paramiko.SSHClient()
self.ssh_client.load_system_host_keys()
self.available_nodes = self.orchastrator_config["available_nodes"]
self.platform_nodes = self.orchastrator_config["platform_nodes"]
# self.user = self.orchastrator_config["user"]
# self.password = self.orchastrator_config["password"]
# self.master_nodes = self.orchastrator_config["master_nodes"]
self.__master = self.orchastrator_config["master"]
self.__token = self.orchastrator_config["token"]
self.user = "******"
self.password = "******"
self.master_nodes = "None"
class InitializeConfig():
def __init__(self, key_prefix, status_path):
self.hostname = os.uname()[1]
logs = Logger(filename = "initialize_config", \
logger_name = "Config Initialize", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
self.key_pref = key_prefix
self.status_path = status_path
logs.clear_handler()
def check_affected(self, val):
command_list = []
logs = Logger(filename = "occonfman", \
logger_name = "Config Initialize check_affected", \
dirname="/aux1/occonfman/")
try:
json_val = json.loads(val)
except:
logs.error(f"Invalid json: {val} ")
return []
if self.hostname in json_val["commands"].keys():
command_list += json_val["commands"][self.hostname]
logs.clear_handler()
return command_list
def apply_preset_configs(self, json_path, etcd_path, reset):
logs = Logger(filename = "occonfman", \
logger_name = "Config Initialize apply_preset_configs", \
dirname="/aux1/occonfman/")
app_path = Path(json_path)
json_data = []
if app_path.is_dir():
for confs in app_path.glob('*.json'):
if confs.is_file():
with open(confs, "r") as conf:
try:
conf_content = conf.read()
json_data += json.loads(conf_content)
etcd_fullpath = etcd_path.rstrip('/') + '/' + str(
confs.name)
self.etcd.write(etcd_fullpath, conf_content)
except:
print(f"Could not load {confs} file")
logs.error(f"Cound not load {confs} file")
continue
else:
print(f"Filepath does not exist: {json_path}")
logs.error(f"Filepath does not exist: {json_path}")
configdict = {}
for cnt in json_data:
if cnt["config"] in configdict.keys():
for arrv in cnt["command"]:
configdict[cnt["config"]].append(arrv)
else:
configdict[cnt["config"]] = list()
for arrv in cnt["command"]:
configdict[cnt["config"]].append(arrv)
for config_data in json_data:
config_key_path = self.key_pref.rstrip('/') + config_data["config"]
config_key = self.etcd.get_key(config_key_path)
if (config_key[0]):
try:
json_content = json.loads(config_key[0].decode("utf-8"))
except:
print("Invalid json for: {} \n Content: {}".format(
config_key_path, config_key[0].decode("utf-8")))
logs.error(
f"Invalid json for: {config_key_path} \n Content: {config_key[0].decode('utf-8')} "
)
continue
if self.hostname in json_content["commands"].keys(
) and not reset:
print(
"Hostname already exists in commands and reset is not set, so no overwrite"
)
logs.info(
"Hostname already exists in commands and reset is not set, so no overwrite"
)
else:
json_content["commands"][self.hostname] = configdict[
config_data["config"]]
self.etcd.write(config_key_path, json_content)
print(
f"Key exists, but {self.hostname} is not in commands or reset is given"
)
logs.info(
f"Key exists, but {self.hostname} is not in commands or reset is given"
)
else:
print(
"Found config file without a key in etcd. Attempting to generate it from template."
)
logs.error(
"Found config file without a key in etcd. Attempting to generate it from template."
)
if Path(config_data["config"]).is_file():
fcontent = ''
json_content = {}
with open(config_data["config"], "r") as content_file:
fcontent = content_file.read()
json_content["content"] = fcontent
json_content["commands"] = dict()
json_content["commands"][self.hostname] = configdict[
config_data["config"]]
json_content["path"] = config_data["config"]
self.etcd.write(config_key_path, json.dumps(json_content))
logs.clear_handler()
def preconfigure(self):
logs = Logger(filename = "occonfman", \
logger_name = "Config Initializa - preconfigure", \
dirname="/aux1/occonfman/")
commands_set = set()
for kvmeta in self.etcd.get_prefix_real(self.key_pref):
ckey = kvmeta[1].key.decode('utf-8')
cval = kvmeta[0].decode("utf-8")
affected_commands = self.check_affected(cval)
if not affected_commands:
print(f"{ckey} does not concern my hostname {self.hostname}")
logs.info(
f"{ckey} does not concern my hostname {self.hostname}")
continue
process_config = ProcessConfig(kvmeta, self.status_path, 1)
process_config.process_config()
# for single_command in affected_commands:
# commands_set.add(single_command)
# for command in commands_set:
# subprocess.run(str(command), shell = True)
logs.clear_handler()
class ProcessConfig():
"""
- Processes a given etcd config key:
- checks if local node is affected by this config change
- generates new config file
- replaces markers with their respective values
- executes the set of commands related to this host
"""
def __init__(self, event, status_path, initial):
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
self.initial = initial
if (self.initial):
self.event_value = json.loads(event[0].decode("utf-8"))
self.event_revision = str(
self.etcd.get_mod_revision(event[1].key.decode("utf-8")))
print(self.event_revision)
log.info(f"Event revision is: {self.event_revision}")
else:
self.event_value = json.loads(event.value)
self.event_revision = event.mod_revision
self.hostname = os.uname()[1]
self.status_path = status_path
log.clear_handler()
def check_affected(self, hosts):
"""
If node hostname is not within
the etcd config's command keys,
then the config change is ignored.
"""
if self.hostname in hosts.keys():
return 0
return 1
def process_config(self):
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
if (self.check_affected(self.event_value["commands"])):
print('This change does not concern my host: {}'.format(
self.hostname))
log.info("This config change does not concern my host {}".format(
self.hostname))
log.clear_handler()
return ''
log.info("Config change: {}".format(self.event_value["path"]))
config_path = self.event_value["path"]
content = self.apply_markers(self.event_value["content"])
self.write_config(config_path, content)
res = self.execute_command()
log.clear_handler()
return (res)
def apply_markers(self, content):
"""
Using jinja2 template engine to replace markers within the config content
"""
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
content_ready = content
if "markers" in self.event_value.keys():
for host in self.event_value["markers"]:
if self.hostname in self.event_value["markers"].keys():
template = jinja2.Template(content)
log.info("Replacing markers for {}".format(self.hostname))
log.clear_handler()
content_ready = template.render(
self.event_value["markers"][self.hostname])
log.clear_handler()
return content_ready
def write_config(self, config_path, content):
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
try:
with open(config_path, 'w') as conf:
conf.write(content)
conf.close()
except:
print(f"Could not write config file: { config_path }")
log.error("Could not write config file {}".format(config_path))
log.clear_handler()
def execute_command(self):
"""
Executes all commands found in commands object
Returns the output of executed commands
"""
results = {}
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
if self.hostname in self.event_value["commands"].keys():
for command in self.event_value["commands"][self.hostname]:
log.info("Executing command {}".format(command))
res = run(command,
stdout=PIPE,
stderr=PIPE,
universal_newlines=True,
shell=True)
log.info("Command output: {}".format(res.stdout))
log.clear_handler()
results[command] = res.stdout
self.return_status(results)
log.clear_handler()
return results
def return_status(self, results):
"""
Writes status key in etcd
Status key containes the executed command output as value
"""
log = Logger(filename = "confman", \
logger_name = "Process Config", \
dirname="/aux1/occonfman/")
stat_path = self.status_path.rstrip(
'/') + self.event_value["path"] + '/' + str(
self.event_revision) + '/' + self.hostname
print(stat_path)
log.info("Writing status key: {} , value: {}".format(
stat_path, results))
log.clear_handler()
self.etcd.write(stat_path, str(results))
class SwarmManagment():
"""
Swarm manager class
"""
def __init__(self):
"""
Constructor of swarm manager
Args:
available_nodes(list)
platform_nodes(list)
user(str)
password(str)
master_node(str)
token(str)
"""
###orchastrator.json way
# self.ssh_client = paramiko.SSHClient()
# self.ssh_client.load_system_host_keys()
# self.available_nodes = parse_config("orchastrator.json")["available_nodes"]
# self.platform_nodes = parse_config("orchastrator.json")["platform_nodes"]
# self.user = parse_config("orchastrator.json")["user"]
# self.password = parse_config("orchastrator.json")["password"]
# self.master_nodes = parse_config("orchastrator.json")["master_nodes"]
# self.__master = parse_config("orchastrator.json")["master"]
# self.__token = parse_config("orchastrator.json")["token"]
###orchastrator.json way
###etcd way
self.etcd_manager = EtcdManagement()
self.orchastrator_config = self.etcd_manager.get_etcd_orchestrator_config()['platform']['orchestrator']
self.ssh_client = paramiko.SSHClient()
self.ssh_client.load_system_host_keys()
self.available_nodes = self.orchastrator_config["available_nodes"]
self.platform_nodes = self.orchastrator_config["platform_nodes"]
# self.user = self.orchastrator_config["user"]
# self.password = self.orchastrator_config["password"]
# self.master_nodes = self.orchastrator_config["master_nodes"]
self.__master = self.orchastrator_config["master"]
self.__token = self.orchastrator_config["token"]
self.user = "******"
self.password = "******"
self.master_nodes = "None"
# self.user = parse_config("orchastrator.json")["user"]
# self.password = parse_config("orchastrator.json")["password"]
# self.master_nodes = parse_config("orchastrator.json")["master_nodes"]
###etcd way
@staticmethod
def get_docker_api(host_ip):
"""
Get docker api client
Args:
host_ip(str)
"""
return docker.DockerClient(base_url='tcp://{}:2375'.format(host_ip))
def add_server(self, host_ips):
"""
Add server to available_nodes
If the server consist in the self.available_nodes
it won't be add
Args:
host_ips(list or str)
Returns:
Append to self.available_nodes the host_ips
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment add_server", dirname="/aux1/ocorchestrator/")
if isinstance(host_ips, str):
if host_ips not in self.available_nodes:
self.available_nodes.append(host_ips)
###orchastrator.json way
# update_config("orchastrator.json", "available_nodes", host_ips, state='add')
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/available_nodes/{}".format(host_ips), "")
###etcd way
else:
# print("The host ip is already in the list")
logger.info("The host ip is already in the list")
logger.clear_handler()
elif isinstance(host_ips, list):
self.available_nodes = list(set(self.available_nodes + host_ips))
###orchastrator.json way
# update_config("orchastrator.json", "available_nodes", host_ips, state='add')
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/available_nodes/{}".format(host_ips), "")
###etcd way
else:
logger.error("Server should be list or string")
logger.clear_handler()
raise TypeError("Server should be list or string")
def add_swarm_server(self, host_ip):
"""
Add server to platform_nodes
If the server consist in the list it won't be add
Args:
host_ips(str)
Returns:
Append to self.platform_nodes the host_ip
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment add_swarm_server", dirname="/aux1/ocorchestrator/")
if isinstance(host_ip, str):
if host_ip not in self.platform_nodes:
self.platform_nodes.append(host_ip)
###orchastrator.json way
# update_config("orchastrator.json", "platform_nodes", host_ip, state='add')
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/platform_nodes/{}".format(host_ip), "")
###etcd way
else:
# print("The host ip is already in the list")
logger.info("The host ip is already in the list")
logger.clear_handler()
def list_available_nodes(self):
"""
List the available servers remain
Returns:
self.available_nodes(list)
"""
###orchastrator.json way
# return parse_config("orchastrator.json")["available_nodes"]
###orchastrator.json way
###etcd way
return self.orchastrator_config["available_nodes"]
###etcd way
def list_platform_nodes(self):
"""
List the servers in the swarm
Returns:
self.platform_nodes(list)
"""
###orchastrator.json way
# return parse_config("orchastrator.json")["platform_nodes"]
###orchastrator.json way
###etcd way
return self.orchastrator_config["platform_nodes"]
###etcd way
def remove_available_server(self, host_ip):
"""
Remove server ip from self.available_nodes
Args:
host_ip(str)
"""
self.available_nodes.remove(host_ip)
###orchastrator.json way
# update_config("orchastrator.json", "available_nodes", host_ip, state='remove')
###orchastrator.json way
###etcd way
self.etcd_manager.remove_key("/orchastrator/available_nodes/{}".format(host_ip))
###etcd way
def remove_swarm_server(self, host_ip):
"""
Remove server ip from self.platform_nodes
Args:
host_ip(str)
"""
if host_ip in self.platform_nodes:
self.platform_nodes.remove(host_ip)
###orchastrator.json way
# update_config("orchastrator.json", "platform_nodes", host_ip, state='remove')
###orchastrator.json way
###etcd way
self.etcd_manager.remove_key("/orchastrator/platform_nodes/{}".format(host_ip))
###etcd way
else:
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment remove_swarm_server", dirname="/aux1/ocorchestrator/")
logger.error("Node {} can't be removed from platform_nodes (It is not in platform_nodes)".format(host_ip))
logger.clear_handler()
def join_server_swarm(self, host_ip):
"""
Join server to the swarm
Args:
host_ip(str)
"""
#####First way
# self.ssh_client.connect(host_ip, username=self.user, password=self.password)
# _, stdout, _ = self.ssh_client.exec_command('docker swarm join --token {} {}:2377'. \
# format(self.__token, self.__master))
# stdout = '\n'.join(map(lambda x: x.rstrip(), stdout.readlines()))
# if re.search(r'This node joined a swarm as a worker', stdout, re.I|re.S):
# self.remove_available_server(host_ip)
# self.add_swarm_server(host_ip)
# else:
# return "Node {} can't be joined to the swarm".format(host_ip)
#####Second way
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment join_server_swarm", dirname="/aux1/ocorchestrator/")
docker_api = self.get_docker_api(host_ip)
response = False
try:
###orchastrator.json way
# response = docker_api.swarm.join(remote_addrs= \
# [parse_config("orchastrator.json")["master"]], \
# join_token = parse_config("orchastrator.json")["token"])
###orchastrator.json way
###etcd way
response = docker_api.swarm.join(remote_addrs= \
[self.orchastrator_config["master"]], \
join_token = self.orchastrator_config["token"])
###etcd way
except docker.errors.APIError as e:
logger.info("Exception handling swarm joining but config will be updated and corrected")
logger.clear_handler()
self.remove_available_server(host_ip)
self.add_swarm_server(host_ip)
if response == True:
logger.info("Node {} was successfully joined to the swarm".format(host_ip))
logger.clear_handler()
self.remove_available_server(host_ip)
self.add_swarm_server(host_ip)
else:
logger.error("Node {} can't be joined to the swarm".format(host_ip))
logger.clear_handler()
return "Node {} can't be joined to the swarm".format(host_ip)
#####Second way
def leave_server_swarm(self, host_ip):
"""
Leave server from the swarm
Args:
host_ip(str)
"""
#####First way
# if host_ip in parse_config("orchastrator.json")["master_nodes"]:
# print("Demoting the node from manager")
# self.demote_manager(host_ip)
# self.ssh_client.connect(host_ip, username=self.user, password=self.password)
# _, stdout, _ = self.ssh_client.exec_command('docker swarm leave')
# stdout = '\n'.join(map(lambda x: x.rstrip(), stdout.readlines()))
# print("STDOUT => {}".format(stdout))
# stdout = "Node left the swarm"
# hostname = self.get_hostname(host_ip)
# if re.search(r'Node left the swarm', stdout, re.I|re.S):
# print("YEEEEE")
# self.ssh_client.connect(self.__master, username=self.user, password=self.password)
# _, leave_stdout, _ = self.ssh_client.exec_command('docker node rm -f {}'.format(hostname))
# leave_stdout = '\n'.join(map(lambda x: x.rstrip(), leave_stdout.readlines()))
# self.add_server(host_ip)
# self.remove_swarm_server(host_ip)
# else:
# return "Node {} can't left the swarm for some reason".format(host_ip)
#####Second way
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment leave_server_swarm", dirname="/aux1/ocorchestrator/")
docker_api = self.get_docker_api(host_ip)
response = docker_api.swarm.leave(force=True)
if response:
self.add_server(host_ip)
self.remove_swarm_server(host_ip)
else:
logger.error("Node {} can't left the swarm for some reason".format(host_ip))
logger.clear_handler()
return "Node {} can't left the swarm for some reason".format(host_ip)
def add_master_node(self, host_ip):
"""
Add server ip to self.master_nodes
Args:
host_ip(str)
"""
self.master_nodes.append(host_ip)
###orchastrator.json way
update_config("orchastrator.json", "master_nodes", host_ip, state='add')
###orchastrator.json way
def remove_master_node(self, host_ip):
"""
Remove server ip to self.master_nodes
Args:
host_ip(str)
"""
self.master_nodes.remove(host_ip)
###orchastrator.json way
update_config("orchastrator.json", "master_nodes", host_ip, state='remove')
###orchastrator.json way
def promote_to_manager(self, host_ip):
"""
Promote the server to manager in the swarm
Args:
host_ip(str)
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment promote_to_manager", dirname="/aux1/ocorchestrator/")
hostname = self.get_hostname(host_ip)
self.ssh_client.connect(self.__master, username=self.user, password=self.password)
_, promoted_stdout, _ = self.ssh_client.exec_command('docker node promote {}'.format(hostname))
promoted_stdout = '\n'.join(map(lambda x: x.rstrip(), promoted_stdout.readlines()))
if re.search(r'promoted to a manager in the swarm', promoted_stdout, re.I|re.S):
self.add_master_node(host_ip)
else:
logger.error("Node {} can't be promoted to manager".format(host_ip))
logger.clear_handler()
return "Node {} can't be promoted to manager".format(host_ip)
def demote_manager(self, host_ip):
"""
Demote the server from manager in the swarm
Args:
host_ip(str)
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment demote_manager", dirname="/aux1/ocorchestrator/")
hostname = self.get_hostname(host_ip)
self.ssh_client.connect(self.__master, username=self.user, password=self.password)
_, demoted_stdout, _ = self.ssh_client.exec_command('docker node demote {}'.format(hostname))
demoted_stdout = '\n'.join(map(lambda x: x.rstrip(), demoted_stdout.readlines()))
if re.search(r'demoted in the swarm', demoted_stdout, re.I|re.S):
self.remove_master_node(host_ip)
else:
logger.error("Node {} can't be demoted from manager".format(host_ip))
logger.clear_handler()
return "Node {} can't be demoted from manager".format(host_ip)
def get_hostname(self, host_ip):
"""
Take the hostname of the server ip
Args:
host_ip(str)
Returns:
hostname(str)
"""
self.ssh_client.connect(host_ip, username=self.user, password=self.password)
_, hostname, _ = self.ssh_client.exec_command('hostname')
hostname = '\n'.join(map(lambda x: x.rstrip(), hostname.readlines()))
return hostname.strip()
def change_master(self, host_ip):
"""
Change the self.__master
Args:
host_ip(str)
"""
self.__master = host_ip
###orchastrator.json way
# update_config("orchastrator.json", "master", host_ip, state="add")
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/master", host_ip)
###etcd way
def change_token(self, token):
"""
Change the self.__token
Args:
token(str)
"""
self.__token = token
###orchastrator.json way
# update_config("orchastrator.json", "token", token, state="add")
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/token", token)
###etcd way
class DecisionMaker():
def __init__(self):
"""
Constructor
Args:
available_nodes(list)
"""
###etcd way
self.etcd_manager = EtcdManagement()
###etcd way
@staticmethod
def get_docker_api(host_ip):
"""
Get docker api client
Args:
host_ip(str)
"""
return docker.DockerClient(base_url='tcp://{}:2375'.format(host_ip))
@staticmethod
def list_containers_by_host(host_ip):
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker list_containers_by_host", \
dirname="/aux1/ocorchestrator/")
docker_api = DecisionMaker.get_docker_api(host_ip)
cont_names = []
try:
for container in docker_api.containers.list():
app_name_search = re.search('(.*?)\_\d+', container.name)
if app_name_search:
app_name = app_name_search.group(1)
cont_names.append(app_name)
except:
logger.error("Can't retrieve data from {} host!".format(host_ip))
### 2019.09.05
if isinstance(host_ip, str):
DecisionMaker().etcd_manager.write("/platform/orchestrator/failed_nodes/{}". \
format(host_ip), "1")
DecisionMaker().etcd_manager.remove_key( \
"/platform/orchestrator/platform_nodes/{}".format(host_ip))
logger.info(
"Moving host {} from platform_nodes to failed_nodes".format(
host_ip))
### 2019.09.05
logger.clear_handler()
logger.clear_handler()
return cont_names
def update_platform_status(self):
logger_2 = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker update_platform_status", \
dirname="/aux1/ocorchestrator/")
names_by_hosts = {}
###etcd way
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
for host in orchestrator_conf['platform_nodes']:
###etcd way
names_by_hosts[host] = {}
try:
docker_api = DecisionMaker.get_docker_api(host)
for container in docker_api.containers.list():
app_name_search = re.search('(.*?)\_\d+', container.name)
if app_name_search:
app_name = app_name_search.group(1)
if app_name not in names_by_hosts[host]:
names_by_hosts[host][app_name] = {}
names_by_hosts[host][app_name].update( \
{container.name: orchestrator_conf['types_instances'][app_name][container.name]})
except:
logger_2.error(
"Can't establish connection with {} host!".format(host))
self.etcd_manager.write("/platform/orchestrator/platform_status",
str(names_by_hosts))
logger_2.clear_handler()
return names_by_hosts
def take_containers_by_hosts(self):
names_by_hosts = {}
###orchastrator.json way
# for host in parse_config('orchastrator.json')['platform_nodes']:
###orchastrator.json way
###etcd way
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
for host in orchestrator_conf['platform_nodes']:
###etcd way
names_by_hosts[host] = dict(
Counter(self.list_containers_by_host(host)))
return names_by_hosts
def counting_app_by_host(self, application):
"""
Counting application by hosts
Args:
application(str)
Returns:
container_count(str)
"""
apps_by_hosts = self.take_containers_by_hosts()
container_count = {}
for host in apps_by_hosts.keys():
if application not in apps_by_hosts[host]:
# return host
container_count[host] = {application: 0}
else:
container_count[host] = {
application: apps_by_hosts[host][application]
}
return container_count
def calculating_app_on_hosts(self):
"""
Args:
None
Returns:
app_counts(dict)
"""
app_counts = {}
for app in self.etcd_manager.get_application_instances():
app_count = self.counting_app_by_host(app)
number = 0
for host in app_count:
number += app_count[host][app]
app_counts[app] = number
return app_counts
def check_for_releasing_node(self):
"""
Check for finding a node that can
be released if it is not necessary
Args:
None
Returns:
host_for_release(str) or None
"""
thresholds = literal_eval(
self.etcd_manager.read_key("/platform/orchestrator/thresholds"))
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
apps_count = self.calculating_app_on_hosts()
curr_nodes_number = len(orchestrator_conf['platform_nodes'])
validation_flag = True
for app in apps_count.keys():
app_count = apps_count[app]
app_per_node = '{}_per_node'.format(app)
if curr_nodes_number*thresholds[app_per_node] - app_count >= \
thresholds[app_per_node]:
pass
else:
validation_flag = False
if validation_flag:
# return "Should be released some node"
all_app_count = 1000
names_by_hosts = self.take_containers_by_hosts()
for host in names_by_hosts.keys():
if host == orchestrator_conf['master']:
continue
curr_count = 0
for app in names_by_hosts[host].keys():
curr_count += names_by_hosts[host][app]
if curr_count < all_app_count:
host_for_release = host
all_app_count = curr_count
return host_for_release
else:
return None
def making_host_decision(self, application, decision, release_node=False):
"""
Make decision on which host to run container
Args:
application(str)
decision(str)
Returns:
host(str)
"""
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
thresholds = literal_eval(
self.etcd_manager.read_key("/platform/orchestrator/thresholds"))
# swarm_manager = SwarmManagment()
app_per_node = "{}_per_node".format(application)
app_by_hosts = self.counting_app_by_host(application)
if release_node:
del (app_by_hosts[release_node])
host_number = len(app_by_hosts.keys())
if decision is 'up':
application_number = 0
for host in app_by_hosts.keys():
if app_by_hosts[host][application] == 0:
return host
else:
application_number += app_by_hosts[host][application]
average_app_number = application_number / host_number
logger_2 = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker making_host_decision", \
dirname="/aux1/ocorchestrator/")
logger_2.info("Aplication {} ||| Average => {}\tApp_per_node => {}". \
format(application, average_app_number, thresholds[app_per_node]))
logger_2.clear_handler()
###logic for adding node to the swarm
if average_app_number >= float(thresholds[app_per_node]):
if len(list(orchestrator_conf['available_nodes'].keys())) != 0:
available_nodes = list(
orchestrator_conf['available_nodes'].keys())
new_node = available_nodes[0]
self.etcd_manager.remove_key("/platform/orchestrator/available_nodes/{}". \
format(new_node))
self.etcd_manager.write("/platform/orchestrator/platform_nodes/{}". \
format(new_node), '1')
return new_node
else:
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker making_host_decision", \
dirname="/aux1/ocorchestrator/")
logger.critical(
"There are not any available servers should"
"look at host stat to run on the lowest"
"loaded host a container")
logger.clear_handler()
###logic for adding node to the swarm
for host in app_by_hosts.keys():
if app_by_hosts[host][application] < average_app_number and \
app_by_hosts[host][application] < float(thresholds[app_per_node]): #parse_config('orchastrator.json')[app_per_node]:
return host
for host in app_by_hosts.keys():
return host
elif decision is 'down':
application_number = 0
for host in app_by_hosts.keys():
application_number += app_by_hosts[host][application]
min_app = "{}_min".format(application)
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker making_host_decision", \
dirname="/aux1/ocorchestrator/")
logger.warning("Application => {}\tmin_apps on platform=> {}\tcurrent app_num {}". \
format(application, thresholds[min_app], application_number))
logger.clear_handler()
if application_number == float(thresholds[min_app]):
return None
average_app_number = application_number / host_number
for host in app_by_hosts.keys():
if app_by_hosts[host][application] > average_app_number and \
app_by_hosts[host][application] < thresholds[app_per_node]: #parse_config('orchastrator.json')[app_per_node]:
return host
for host in app_by_hosts.keys():
return host
def release_node(self, host):
"""
Stop all containers from the passed node,
move them to the other hosts in self.platform_nodes,
and move the host to available.servers
Args:
host(str)
Returns:
None
"""
container_manager = ContainerManagement()
apps_by_host = container_manager.get_container_names_by_host(host)
for app in apps_by_host:
app_name_search = re.search('(.*?)\_\d+', app)
if app_name_search:
app_name = app_name_search.group(1)
container_manager.stop_container(name=app, host_ip=host)
new_host = self.making_host_decision(application=app_name, \
decision='up', \
release_node=host)
container_manager.run_container_name(host_ip=new_host, \
application=app_name, \
container_hostname=app)
#####
self.etcd_manager.remove_key(
"/platform/orchestrator/platform_nodes/{}".format(host))
self.etcd_manager.write(
"/platform/orchestrator/available_nodes/{}".format(host), '1')
#####
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker release_node", \
dirname="/aux1/ocorchestrator/")
logger.warning("Releasing node {} was successfull !".format(host))
logger.clear_handler()
def __init__(self): self.current_md5state = "" self.current_platform_state = "" self.current_state_hosts = [] self.etcd_manager = EtcdManagement()
def __init__(self, confman_etcd_root):
threading.Thread.__init__(self)
logs = Logger(filename = "watcher", \
logger_name = "Config Watcher", \
dirname="/aux1/occonfman/")
self.etcd = EtcdManagement()
normalized_confman_etcd_root = os.path.normpath(confman_etcd_root)
confman_config = f"{normalized_confman_etcd_root}/occonfman"
self.confman_key = {}
self.conf = []
try:
self.conf = self.etcd.get_key(confman_config)
print("Getting main config for occonfman")
logs.info("Getting main config for occonfman")
if (self.conf[0]):
self.confman_key = json.loads(self.conf[0].decode("utf-8"))
logs.info(f"Successfully retrieved main config for occonfman")
else:
logs.error(f"Confman config may be empty {self.conf}")
except Exception as e:
print(
f"Watcher was unable to retrieve its main config file {confman_config}"
)
logs.log.exception(
f"Watcher was unable to get its config file {confman_config}")
raise (e)
# At this point we know that ectd is available
# if self.confman_key is not present or it is empty, add default config to ectd
if not self.confman_key:
self.confman_key = {
"config_key_prefix":
f"{normalized_confman_etcd_root}/configs/",
"status_key_prefix": f"{normalized_confman_etcd_root}/status/",
"appconf_json_key":
f"{normalized_confman_etcd_root}/occonfman/",
"appconf_json_dir": "/aux0/customer/occonfman/"
}
logs.warning(
f"{confman_config} is not present in etcd or empty. Using default: {self.confman_key}"
)
self.etcd.write(confman_config, json.dumps(self.confman_key))
"""
Initialize the configurations
- read all application json config files and copy them to etcd
- check if config key exists, if not, copy its content and commands
- run through all configs and process them one by one
"""
logs.info("Starting config initialization procedure.")
if (self.confman_key):
self.initial = InitializeConfig(
self.confman_key["config_key_prefix"],
self.confman_key["status_key_prefix"])
logs.info("Applying preset configs for initialization:")
self.initial.apply_preset_configs(
self.confman_key["appconf_json_dir"],
self.confman_key["appconf_json_key"], 0)
logs.info(
"Going through each config file in etcd and processing its content and commands:"
)
self.initial.preconfigure()
else:
logs.error(
f"No initialization will be done, since main confman key was not present: {confman_config}"
)
print(
f"No initialization, because there's no confman key: {confman_config}"
)
logs.clear_handler()
class ConfigSupervisor():
def __init__(self):
self.current_md5state = ""
self.current_platform_state = ""
self.current_state_hosts = []
self.etcd_manager = EtcdManagement()
def check_nodesxml(self):
platform_nodes = self.etcd_manager.get_etcd_orchestrator_config()
platform_state = ast.literal_eval(platform_nodes["platform"]["orchestrator"]["platform_status"])
mapping = json.loads(platform_nodes["platform"]["orchestrator"]["nodesxml_mapping"])
md5state = hashlib.md5(str(platform_state).encode('utf-8')).hexdigest()
state_hosts = []
if(md5state == self.current_md5state):
# write to log here
pass
else:
logger = Logger(filename = "orchestrator", \
logger_name="ConfigSupervisor check_nodesxml", \
dirname="/aux1/ocorchestrator/")
hostnames = self.get_nodexml_hostnames(platform_nodes["platform"]["orchestrator"]["nodes_xml"])
state_hosts = self.get_hosts(platform_state,state_hosts)
host_diff = self.is_container_in_nodesxml(hostnames,state_hosts)
logger.warning("Hostnames: {}, State hosts: {} Host_diff: {}".format(hostnames,state_hosts,host_diff))
if(host_diff):
self.add_hostnames_to_nodesxml(platform_nodes["platform"]["orchestrator"]["nodes_xml"],host_diff)
added,removed = self.compare_hostnames_in_platform_states(state_hosts,self.current_state_hosts)
self.add_nodeid_to_nodesxml_group(platform_nodes["platform"]["orchestrator"]["nodes_xml"],added,removed,mapping)
self.current_md5state = md5state
self.current_platform_state = platform_state
self.current_state_hosts = state_hosts
logger.clear_handler()
return 0
def get_hosts(self, dat, ips = []):
for k,v in dat.items():
if(type(v) is dict):
self.get_hosts(v, ips)
else:
ips.append(k)
return ips
def get_ips_of_servers(self, orc_state):
server_ips = []
for i in orc_state.keys():
if(re.match(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}',i)):
server_ips.append(i)
else:
pass
return server_ips
def get_nodexml_hostnames(self, nxml):
nodes_xml = xmltodict.parse(nxml,process_namespaces=True)
ba = []
# self.logger.warning("type {} value: {}".format(type(nodes_xml),nodes_xml))
try:
for k in nodes_xml["nodesinfo"]["nodes"]["node"]:
# self.logger.warning("Type of nodesxml is: {}, value is: {}".format(type(k),k))
# print(k)
ba.append(k["@hostname"])
except:
pass
# self.logger.clear_handler()
return ba
def is_container_in_nodesxml(self, nodes,states):
return list(set(states).difference(nodes))
def compare_hostnames_in_platform_states(self, newstate,oldstate):
added = list(set(newstate).difference(oldstate))
removed = list(set(oldstate).difference(newstate))
return added,removed
def get_max_nodesxml_id(self, nodesxml):
nodes_xml = xmltodict.parse(nodesxml,process_namespaces=True)
cn = 0
try:
for k in nodes_xml["nodesinfo"]["nodes"]["node"]:
ci = int(k["@id"])
if(cn < ci):
cn = ci
except:
return 1
return cn
def add_hostnames_to_nodesxml(self, nodesxml,hostnames):
max_node_id = self.get_max_nodesxml_id(nodesxml)
nodes_xml = nodesxml
for host in hostnames:
max_node_id += 1
node = r'<node id="'+str(max_node_id)+'" hostname="'+host+'" /></nodes>'
nodes_xml = re.sub(r'<\/nodes>',node,nodes_xml)
self.etcd_manager.write("/platform/orchestrator/nodes_xml",nodes_xml)
return nodes_xml
def get_nodeid_by_hostname(self, nodesxml,hostname):
ba = 0
for k in nodesxml["nodesinfo"]["nodes"]["node"]:
if(k["@hostname"] == hostname):
ba = k["@id"]
return ba
def add_nodeid_to_nodesxml_group(self, nodesxml,for_insert,for_removal,mapping):
platform_nodes = self.etcd_manager.get_etcd_orchestrator_config()
nodes_xml = xmltodict.parse(platform_nodes["platform"]["orchestrator"]["nodes_xml"],process_namespaces=True)
new_nodes_xml = platform_nodes["platform"]["orchestrator"]["nodes_xml"]
for a in for_insert:
_,pltfm,app_type,id,_ = re.split(r'(^.*?)_(.*?)_.*(\d+)$',a)
id = str(self.get_nodeid_by_hostname(nodes_xml,a))
# print("App is: {} Id: {}".format(app_type,id))
for b in mapping[app_type].split(","):
pattern = r'(?s)(<nodetype\s+id="'+ b +'".*?)<\/nodeslist>'
node_id_pattern = r'(?s)<node\s+id="'+ id +'"\s+\/>'
group_content = re.search(pattern,new_nodes_xml)
if(re.search(node_id_pattern, group_content.group(0))):
continue
replacement = r'\1<node id="'+ id +'" /></nodeslist>'
new_nodes_xml = re.sub(pattern,replacement,new_nodes_xml)
for c in for_removal:
_,_,app_type1,_,_ = re.split(r'(^.*?)_(.*?)_.*(\d+)$',c)
id1 = str(self.get_nodeid_by_hostname(nodes_xml,c))
# print("Removal app is: {} Id: {}".format(app_type1,id1))
for d in mapping[app_type1].split(","):
node_id_pattern1 = r'(?s)<node\s+id="'+ id1 +'"\s+\/>'
new_nodes_xml = re.sub(node_id_pattern1,'',new_nodes_xml)
self.etcd_manager.write("/platform/orchestrator/nodes_xml",new_nodes_xml)
return new_nodes_xml
