def delete(self):
parser = reqparse.RequestParser()
data = request.get_json()
resp_list = []
etcd_manager = EtcdManagement()
deleted_flag = None
for key_path in data:
try:
deleted_flag = etcd_manager.remove_key(key_path)
except:
resp_list.append({
"key": key_path,
"status": "failed",
"code": 1002
})
continue
if deleted_flag:
resp_list.append({"key": key_path, "status": "success"})
else:
resp_list.append({
"key": key_path,
"status": "failed",
"code": 1002
})
deleted_flag = None
return jsonify(resp_list)
def delete(self):
parser = reqparse.RequestParser()
parser.add_argument('key', type=str)
args = parser.parse_args()
etcd_manager = EtcdManagement()
deleted_flag = None
try:
deleted_flag = etcd_manager.remove_key(args["key"])
except:
return {'message': "Request can't be executed", "code": 1000}, 500
if deleted_flag:
return {'message': 'Key was deleted successfully'}, 200
else:
return {
'message': "Key wasn't deleted successfully",
"code": 1002
}, 409
class DecisionMaker():
def __init__(self):
"""
Constructor
Args:
available_nodes(list)
"""
###etcd way
self.etcd_manager = EtcdManagement()
###etcd way
@staticmethod
def get_docker_api(host_ip):
"""
Get docker api client
Args:
host_ip(str)
"""
return docker.DockerClient(base_url='tcp://{}:2375'.format(host_ip))
@staticmethod
def list_containers_by_host(host_ip):
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker list_containers_by_host", \
dirname="/aux1/ocorchestrator/")
docker_api = DecisionMaker.get_docker_api(host_ip)
cont_names = []
try:
for container in docker_api.containers.list():
app_name_search = re.search('(.*?)\_\d+', container.name)
if app_name_search:
app_name = app_name_search.group(1)
cont_names.append(app_name)
except:
logger.error("Can't retrieve data from {} host!".format(host_ip))
### 2019.09.05
if isinstance(host_ip, str):
DecisionMaker().etcd_manager.write("/platform/orchestrator/failed_nodes/{}". \
format(host_ip), "1")
DecisionMaker().etcd_manager.remove_key( \
"/platform/orchestrator/platform_nodes/{}".format(host_ip))
logger.info(
"Moving host {} from platform_nodes to failed_nodes".format(
host_ip))
### 2019.09.05
logger.clear_handler()
logger.clear_handler()
return cont_names
def update_platform_status(self):
logger_2 = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker update_platform_status", \
dirname="/aux1/ocorchestrator/")
names_by_hosts = {}
###etcd way
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
for host in orchestrator_conf['platform_nodes']:
###etcd way
names_by_hosts[host] = {}
try:
docker_api = DecisionMaker.get_docker_api(host)
for container in docker_api.containers.list():
app_name_search = re.search('(.*?)\_\d+', container.name)
if app_name_search:
app_name = app_name_search.group(1)
if app_name not in names_by_hosts[host]:
names_by_hosts[host][app_name] = {}
names_by_hosts[host][app_name].update( \
{container.name: orchestrator_conf['types_instances'][app_name][container.name]})
except:
logger_2.error(
"Can't establish connection with {} host!".format(host))
self.etcd_manager.write("/platform/orchestrator/platform_status",
str(names_by_hosts))
logger_2.clear_handler()
return names_by_hosts
def take_containers_by_hosts(self):
names_by_hosts = {}
###orchastrator.json way
# for host in parse_config('orchastrator.json')['platform_nodes']:
###orchastrator.json way
###etcd way
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
for host in orchestrator_conf['platform_nodes']:
###etcd way
names_by_hosts[host] = dict(
Counter(self.list_containers_by_host(host)))
return names_by_hosts
def counting_app_by_host(self, application):
"""
Counting application by hosts
Args:
application(str)
Returns:
container_count(str)
"""
apps_by_hosts = self.take_containers_by_hosts()
container_count = {}
for host in apps_by_hosts.keys():
if application not in apps_by_hosts[host]:
# return host
container_count[host] = {application: 0}
else:
container_count[host] = {
application: apps_by_hosts[host][application]
}
return container_count
def calculating_app_on_hosts(self):
"""
Args:
None
Returns:
app_counts(dict)
"""
app_counts = {}
for app in self.etcd_manager.get_application_instances():
app_count = self.counting_app_by_host(app)
number = 0
for host in app_count:
number += app_count[host][app]
app_counts[app] = number
return app_counts
def check_for_releasing_node(self):
"""
Check for finding a node that can
be released if it is not necessary
Args:
None
Returns:
host_for_release(str) or None
"""
thresholds = literal_eval(
self.etcd_manager.read_key("/platform/orchestrator/thresholds"))
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
apps_count = self.calculating_app_on_hosts()
curr_nodes_number = len(orchestrator_conf['platform_nodes'])
validation_flag = True
for app in apps_count.keys():
app_count = apps_count[app]
app_per_node = '{}_per_node'.format(app)
if curr_nodes_number*thresholds[app_per_node] - app_count >= \
thresholds[app_per_node]:
pass
else:
validation_flag = False
if validation_flag:
# return "Should be released some node"
all_app_count = 1000
names_by_hosts = self.take_containers_by_hosts()
for host in names_by_hosts.keys():
if host == orchestrator_conf['master']:
continue
curr_count = 0
for app in names_by_hosts[host].keys():
curr_count += names_by_hosts[host][app]
if curr_count < all_app_count:
host_for_release = host
all_app_count = curr_count
return host_for_release
else:
return None
def making_host_decision(self, application, decision, release_node=False):
"""
Make decision on which host to run container
Args:
application(str)
decision(str)
Returns:
host(str)
"""
orchestrator_conf = self.etcd_manager. \
get_etcd_orchestrator_config()['platform']['orchestrator']
thresholds = literal_eval(
self.etcd_manager.read_key("/platform/orchestrator/thresholds"))
# swarm_manager = SwarmManagment()
app_per_node = "{}_per_node".format(application)
app_by_hosts = self.counting_app_by_host(application)
if release_node:
del (app_by_hosts[release_node])
host_number = len(app_by_hosts.keys())
if decision is 'up':
application_number = 0
for host in app_by_hosts.keys():
if app_by_hosts[host][application] == 0:
return host
else:
application_number += app_by_hosts[host][application]
average_app_number = application_number / host_number
logger_2 = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker making_host_decision", \
dirname="/aux1/ocorchestrator/")
logger_2.info("Aplication {} ||| Average => {}\tApp_per_node => {}". \
format(application, average_app_number, thresholds[app_per_node]))
logger_2.clear_handler()
###logic for adding node to the swarm
if average_app_number >= float(thresholds[app_per_node]):
if len(list(orchestrator_conf['available_nodes'].keys())) != 0:
available_nodes = list(
orchestrator_conf['available_nodes'].keys())
new_node = available_nodes[0]
self.etcd_manager.remove_key("/platform/orchestrator/available_nodes/{}". \
format(new_node))
self.etcd_manager.write("/platform/orchestrator/platform_nodes/{}". \
format(new_node), '1')
return new_node
else:
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker making_host_decision", \
dirname="/aux1/ocorchestrator/")
logger.critical(
"There are not any available servers should"
"look at host stat to run on the lowest"
"loaded host a container")
logger.clear_handler()
###logic for adding node to the swarm
for host in app_by_hosts.keys():
if app_by_hosts[host][application] < average_app_number and \
app_by_hosts[host][application] < float(thresholds[app_per_node]): #parse_config('orchastrator.json')[app_per_node]:
return host
for host in app_by_hosts.keys():
return host
elif decision is 'down':
application_number = 0
for host in app_by_hosts.keys():
application_number += app_by_hosts[host][application]
min_app = "{}_min".format(application)
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker making_host_decision", \
dirname="/aux1/ocorchestrator/")
logger.warning("Application => {}\tmin_apps on platform=> {}\tcurrent app_num {}". \
format(application, thresholds[min_app], application_number))
logger.clear_handler()
if application_number == float(thresholds[min_app]):
return None
average_app_number = application_number / host_number
for host in app_by_hosts.keys():
if app_by_hosts[host][application] > average_app_number and \
app_by_hosts[host][application] < thresholds[app_per_node]: #parse_config('orchastrator.json')[app_per_node]:
return host
for host in app_by_hosts.keys():
return host
def release_node(self, host):
"""
Stop all containers from the passed node,
move them to the other hosts in self.platform_nodes,
and move the host to available.servers
Args:
host(str)
Returns:
None
"""
container_manager = ContainerManagement()
apps_by_host = container_manager.get_container_names_by_host(host)
for app in apps_by_host:
app_name_search = re.search('(.*?)\_\d+', app)
if app_name_search:
app_name = app_name_search.group(1)
container_manager.stop_container(name=app, host_ip=host)
new_host = self.making_host_decision(application=app_name, \
decision='up', \
release_node=host)
container_manager.run_container_name(host_ip=new_host, \
application=app_name, \
container_hostname=app)
#####
self.etcd_manager.remove_key(
"/platform/orchestrator/platform_nodes/{}".format(host))
self.etcd_manager.write(
"/platform/orchestrator/available_nodes/{}".format(host), '1')
#####
logger = Logger(filename = "orchestrator", \
logger_name = "DecisionMaker release_node", \
dirname="/aux1/ocorchestrator/")
logger.warning("Releasing node {} was successfull !".format(host))
logger.clear_handler()
class SwarmManagment():
"""
Swarm manager class
"""
def __init__(self):
"""
Constructor of swarm manager
Args:
available_nodes(list)
platform_nodes(list)
user(str)
password(str)
master_node(str)
token(str)
"""
###orchastrator.json way
# self.ssh_client = paramiko.SSHClient()
# self.ssh_client.load_system_host_keys()
# self.available_nodes = parse_config("orchastrator.json")["available_nodes"]
# self.platform_nodes = parse_config("orchastrator.json")["platform_nodes"]
# self.user = parse_config("orchastrator.json")["user"]
# self.password = parse_config("orchastrator.json")["password"]
# self.master_nodes = parse_config("orchastrator.json")["master_nodes"]
# self.__master = parse_config("orchastrator.json")["master"]
# self.__token = parse_config("orchastrator.json")["token"]
###orchastrator.json way
###etcd way
self.etcd_manager = EtcdManagement()
self.orchastrator_config = self.etcd_manager.get_etcd_orchestrator_config()['platform']['orchestrator']
self.ssh_client = paramiko.SSHClient()
self.ssh_client.load_system_host_keys()
self.available_nodes = self.orchastrator_config["available_nodes"]
self.platform_nodes = self.orchastrator_config["platform_nodes"]
# self.user = self.orchastrator_config["user"]
# self.password = self.orchastrator_config["password"]
# self.master_nodes = self.orchastrator_config["master_nodes"]
self.__master = self.orchastrator_config["master"]
self.__token = self.orchastrator_config["token"]
self.user = "******"
self.password = "******"
self.master_nodes = "None"
# self.user = parse_config("orchastrator.json")["user"]
# self.password = parse_config("orchastrator.json")["password"]
# self.master_nodes = parse_config("orchastrator.json")["master_nodes"]
###etcd way
@staticmethod
def get_docker_api(host_ip):
"""
Get docker api client
Args:
host_ip(str)
"""
return docker.DockerClient(base_url='tcp://{}:2375'.format(host_ip))
def add_server(self, host_ips):
"""
Add server to available_nodes
If the server consist in the self.available_nodes
it won't be add
Args:
host_ips(list or str)
Returns:
Append to self.available_nodes the host_ips
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment add_server", dirname="/aux1/ocorchestrator/")
if isinstance(host_ips, str):
if host_ips not in self.available_nodes:
self.available_nodes.append(host_ips)
###orchastrator.json way
# update_config("orchastrator.json", "available_nodes", host_ips, state='add')
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/available_nodes/{}".format(host_ips), "")
###etcd way
else:
# print("The host ip is already in the list")
logger.info("The host ip is already in the list")
logger.clear_handler()
elif isinstance(host_ips, list):
self.available_nodes = list(set(self.available_nodes + host_ips))
###orchastrator.json way
# update_config("orchastrator.json", "available_nodes", host_ips, state='add')
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/available_nodes/{}".format(host_ips), "")
###etcd way
else:
logger.error("Server should be list or string")
logger.clear_handler()
raise TypeError("Server should be list or string")
def add_swarm_server(self, host_ip):
"""
Add server to platform_nodes
If the server consist in the list it won't be add
Args:
host_ips(str)
Returns:
Append to self.platform_nodes the host_ip
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment add_swarm_server", dirname="/aux1/ocorchestrator/")
if isinstance(host_ip, str):
if host_ip not in self.platform_nodes:
self.platform_nodes.append(host_ip)
###orchastrator.json way
# update_config("orchastrator.json", "platform_nodes", host_ip, state='add')
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/platform_nodes/{}".format(host_ip), "")
###etcd way
else:
# print("The host ip is already in the list")
logger.info("The host ip is already in the list")
logger.clear_handler()
def list_available_nodes(self):
"""
List the available servers remain
Returns:
self.available_nodes(list)
"""
###orchastrator.json way
# return parse_config("orchastrator.json")["available_nodes"]
###orchastrator.json way
###etcd way
return self.orchastrator_config["available_nodes"]
###etcd way
def list_platform_nodes(self):
"""
List the servers in the swarm
Returns:
self.platform_nodes(list)
"""
###orchastrator.json way
# return parse_config("orchastrator.json")["platform_nodes"]
###orchastrator.json way
###etcd way
return self.orchastrator_config["platform_nodes"]
###etcd way
def remove_available_server(self, host_ip):
"""
Remove server ip from self.available_nodes
Args:
host_ip(str)
"""
self.available_nodes.remove(host_ip)
###orchastrator.json way
# update_config("orchastrator.json", "available_nodes", host_ip, state='remove')
###orchastrator.json way
###etcd way
self.etcd_manager.remove_key("/orchastrator/available_nodes/{}".format(host_ip))
###etcd way
def remove_swarm_server(self, host_ip):
"""
Remove server ip from self.platform_nodes
Args:
host_ip(str)
"""
if host_ip in self.platform_nodes:
self.platform_nodes.remove(host_ip)
###orchastrator.json way
# update_config("orchastrator.json", "platform_nodes", host_ip, state='remove')
###orchastrator.json way
###etcd way
self.etcd_manager.remove_key("/orchastrator/platform_nodes/{}".format(host_ip))
###etcd way
else:
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment remove_swarm_server", dirname="/aux1/ocorchestrator/")
logger.error("Node {} can't be removed from platform_nodes (It is not in platform_nodes)".format(host_ip))
logger.clear_handler()
def join_server_swarm(self, host_ip):
"""
Join server to the swarm
Args:
host_ip(str)
"""
#####First way
# self.ssh_client.connect(host_ip, username=self.user, password=self.password)
# _, stdout, _ = self.ssh_client.exec_command('docker swarm join --token {} {}:2377'. \
# format(self.__token, self.__master))
# stdout = '\n'.join(map(lambda x: x.rstrip(), stdout.readlines()))
# if re.search(r'This node joined a swarm as a worker', stdout, re.I|re.S):
# self.remove_available_server(host_ip)
# self.add_swarm_server(host_ip)
# else:
# return "Node {} can't be joined to the swarm".format(host_ip)
#####Second way
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment join_server_swarm", dirname="/aux1/ocorchestrator/")
docker_api = self.get_docker_api(host_ip)
response = False
try:
###orchastrator.json way
# response = docker_api.swarm.join(remote_addrs= \
# [parse_config("orchastrator.json")["master"]], \
# join_token = parse_config("orchastrator.json")["token"])
###orchastrator.json way
###etcd way
response = docker_api.swarm.join(remote_addrs= \
[self.orchastrator_config["master"]], \
join_token = self.orchastrator_config["token"])
###etcd way
except docker.errors.APIError as e:
logger.info("Exception handling swarm joining but config will be updated and corrected")
logger.clear_handler()
self.remove_available_server(host_ip)
self.add_swarm_server(host_ip)
if response == True:
logger.info("Node {} was successfully joined to the swarm".format(host_ip))
logger.clear_handler()
self.remove_available_server(host_ip)
self.add_swarm_server(host_ip)
else:
logger.error("Node {} can't be joined to the swarm".format(host_ip))
logger.clear_handler()
return "Node {} can't be joined to the swarm".format(host_ip)
#####Second way
def leave_server_swarm(self, host_ip):
"""
Leave server from the swarm
Args:
host_ip(str)
"""
#####First way
# if host_ip in parse_config("orchastrator.json")["master_nodes"]:
# print("Demoting the node from manager")
# self.demote_manager(host_ip)
# self.ssh_client.connect(host_ip, username=self.user, password=self.password)
# _, stdout, _ = self.ssh_client.exec_command('docker swarm leave')
# stdout = '\n'.join(map(lambda x: x.rstrip(), stdout.readlines()))
# print("STDOUT => {}".format(stdout))
# stdout = "Node left the swarm"
# hostname = self.get_hostname(host_ip)
# if re.search(r'Node left the swarm', stdout, re.I|re.S):
# print("YEEEEE")
# self.ssh_client.connect(self.__master, username=self.user, password=self.password)
# _, leave_stdout, _ = self.ssh_client.exec_command('docker node rm -f {}'.format(hostname))
# leave_stdout = '\n'.join(map(lambda x: x.rstrip(), leave_stdout.readlines()))
# self.add_server(host_ip)
# self.remove_swarm_server(host_ip)
# else:
# return "Node {} can't left the swarm for some reason".format(host_ip)
#####Second way
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment leave_server_swarm", dirname="/aux1/ocorchestrator/")
docker_api = self.get_docker_api(host_ip)
response = docker_api.swarm.leave(force=True)
if response:
self.add_server(host_ip)
self.remove_swarm_server(host_ip)
else:
logger.error("Node {} can't left the swarm for some reason".format(host_ip))
logger.clear_handler()
return "Node {} can't left the swarm for some reason".format(host_ip)
def add_master_node(self, host_ip):
"""
Add server ip to self.master_nodes
Args:
host_ip(str)
"""
self.master_nodes.append(host_ip)
###orchastrator.json way
update_config("orchastrator.json", "master_nodes", host_ip, state='add')
###orchastrator.json way
def remove_master_node(self, host_ip):
"""
Remove server ip to self.master_nodes
Args:
host_ip(str)
"""
self.master_nodes.remove(host_ip)
###orchastrator.json way
update_config("orchastrator.json", "master_nodes", host_ip, state='remove')
###orchastrator.json way
def promote_to_manager(self, host_ip):
"""
Promote the server to manager in the swarm
Args:
host_ip(str)
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment promote_to_manager", dirname="/aux1/ocorchestrator/")
hostname = self.get_hostname(host_ip)
self.ssh_client.connect(self.__master, username=self.user, password=self.password)
_, promoted_stdout, _ = self.ssh_client.exec_command('docker node promote {}'.format(hostname))
promoted_stdout = '\n'.join(map(lambda x: x.rstrip(), promoted_stdout.readlines()))
if re.search(r'promoted to a manager in the swarm', promoted_stdout, re.I|re.S):
self.add_master_node(host_ip)
else:
logger.error("Node {} can't be promoted to manager".format(host_ip))
logger.clear_handler()
return "Node {} can't be promoted to manager".format(host_ip)
def demote_manager(self, host_ip):
"""
Demote the server from manager in the swarm
Args:
host_ip(str)
"""
logger = Logger(filename = "orchastrator", logger_name = "SwarmManagment demote_manager", dirname="/aux1/ocorchestrator/")
hostname = self.get_hostname(host_ip)
self.ssh_client.connect(self.__master, username=self.user, password=self.password)
_, demoted_stdout, _ = self.ssh_client.exec_command('docker node demote {}'.format(hostname))
demoted_stdout = '\n'.join(map(lambda x: x.rstrip(), demoted_stdout.readlines()))
if re.search(r'demoted in the swarm', demoted_stdout, re.I|re.S):
self.remove_master_node(host_ip)
else:
logger.error("Node {} can't be demoted from manager".format(host_ip))
logger.clear_handler()
return "Node {} can't be demoted from manager".format(host_ip)
def get_hostname(self, host_ip):
"""
Take the hostname of the server ip
Args:
host_ip(str)
Returns:
hostname(str)
"""
self.ssh_client.connect(host_ip, username=self.user, password=self.password)
_, hostname, _ = self.ssh_client.exec_command('hostname')
hostname = '\n'.join(map(lambda x: x.rstrip(), hostname.readlines()))
return hostname.strip()
def change_master(self, host_ip):
"""
Change the self.__master
Args:
host_ip(str)
"""
self.__master = host_ip
###orchastrator.json way
# update_config("orchastrator.json", "master", host_ip, state="add")
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/master", host_ip)
###etcd way
def change_token(self, token):
"""
Change the self.__token
Args:
token(str)
"""
self.__token = token
###orchastrator.json way
# update_config("orchastrator.json", "token", token, state="add")
###orchastrator.json way
###etcd way
self.etcd_manager.write("/orchastrator/token", token)
###etcd way