def send_cookie(login):
session_ttl = int()
session_ttl = sql.get_setting('session_ttl')
session_ttl = int(session_ttl)
expires = datetime.datetime.utcnow() + datetime.timedelta(days=session_ttl)
user_uuid = str(uuid.uuid4())
user_token = str(uuid.uuid4())
c = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
c["uuid"] = user_uuid
c["uuid"]["path"] = "/app/"
c["uuid"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
print(c)
sql.write_user_uuid(login, user_uuid)
sql.write_user_token(login, user_token)
try:
funct.logging('locahost',
sql.get_user_name_by_uuid(user_uuid) + ' log in',
haproxywi=1,
login=1)
except:
pass
print("Content-type: text/html\n")
print('ok')
sys.exit()
def send_cookie(login):
session_ttl = int()
session_ttl = sql.get_setting('session_ttl')
session_ttl = int(session_ttl)
expires = datetime.datetime.utcnow() + datetime.timedelta(days=session_ttl)
user_uuid = str(uuid.uuid4())
user_token = str(uuid.uuid4())
sql.write_user_uuid(login, user_uuid)
sql.write_user_token(login, user_token)
id = sql.get_user_id_by_uuid(user_uuid)
user_groups = sql.select_user_groups(id, limit=1)
c = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
c["uuid"] = user_uuid
c["uuid"]["path"] = "/"
# c["uuid"]["sameSite"] = "Strict"
c["uuid"]["Secure"] = "True"
c["uuid"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
c["group"] = user_groups
c["group"]["path"] = "/"
# c["group"]["sameSite"] = "Strict"
c["group"]["Secure"] = "True"
c["group"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
print(c)
try:
funct.logging('locahost',
' ' + sql.get_user_name_by_uuid(user_uuid) + ' log in',
haproxywi=1)
except:
pass
print("Content-type: text/html\n")
print('ok')
sys.exit()
def start_worker(serv):
port = funct.get_config_var('haproxy', 'haproxy_sock_port')
cmd = "tools/checker_worker.py %s --port %s &" % (serv, port)
os.system(cmd)
funct.logging("localhost",
" Masrer started new worker for: " + serv,
alerting=1)
def main():
port = sql.get_setting('haproxy_sock_port')
readstats = ""
killer = GracefulKiller()
while True:
servers = sql.select_keep_alive()
for serv in servers:
try:
readstats = subprocess.check_output(
["echo show stat | nc " + serv[0] + " " + port],
shell=True)
except CalledProcessError as e:
alert = "Try start HAProxy serivce at " + serv[0]
funct.logging("localhost", " " + alert, keep_alive=1)
start_command = []
start_command.append('sudo ' +
sql.get_setting('restart_command'))
funct.ssh_command(serv[0], start_command)
time.sleep(30)
continue
except OSError as e:
print(e)
sys.exit()
else:
cur_stat_service = "Ok"
time.sleep(40)
def start_worker(serv):
port = sql.get_setting('haproxy_sock_port')
cmd = "tools/metrics_worker.py %s --port %s &" % (serv, port)
os.system(cmd)
funct.logging("localhost",
" Masrer started new metrics worker for: " + serv,
metrics=1)
def kill_all_workers():
cmd = "ps ax |grep -e 'tools/metrics_worker.py\|tools/metrics_waf_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost",
" Masrer killing all metrics workers",
metrics=1)
if stderr:
funct.logging("localhost", stderr, metrics=1)
def kill_worker(serv):
cmd = "ps ax |grep 'tools/metrics_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost",
" Masrer killed metrics worker for: " + serv,
metrics=1)
if stderr:
funct.logging("localhost", stderr, metrics=1)
def check_group(group, role_id):
import http.cookies
import os
cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
user_id = cookie.get('uuid')
user_group = get_user_group_by_uuid(user_id.value)
if user_group == group or user_group == '1' or role_id == 1:
return True
else:
funct.logging('localhost', ' has tried to actions in not own group ', haproxywi=1, login=1)
return False
def send_cookie(login):
session_ttl = sql.get_setting('session_ttl')
session_ttl = int(session_ttl)
expires = datetime.datetime.utcnow() + datetime.timedelta(days=session_ttl)
user_uuid = str(uuid.uuid4())
user_token = str(uuid.uuid4())
sql.write_user_uuid(login, user_uuid)
sql.write_user_token(login, user_token)
id = sql.get_user_id_by_uuid(user_uuid)
try:
cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
user_group_id = cookie.get('group')
user_group_id = user_group_id.value
if sql.check_user_group(id, user_group_id):
user_groups = user_group_id
else:
user_groups = sql.select_user_groups(id, limit=1)
except:
user_groups = sql.select_user_groups(id, limit=1)
c = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
c["uuid"] = user_uuid
c["uuid"]["path"] = "/app"
# c["uuid"]["samesite"] = "Strict"
c["uuid"]["Secure"] = "True"
c["uuid"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
c["group"] = user_groups
c["group"]["path"] = "/app"
# c["group"]["samesite"] = "Strict"
c["group"]["Secure"] = "True"
c["group"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
print(c.output())
try:
groups = sql.select_groups(id=user_groups)
for g in groups:
if g[0] == int(user_groups):
user_group = g[1]
except:
user_group = ''
try:
user_name = sql.get_user_name_by_uuid(user_uuid)
funct.logging('localhost',
' user: '******', group: ' + user_group +
' log in',
haproxywi=1)
except:
pass
print("Content-type: text/html\n")
print('ok')
sys.exit()
def ban():
c = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
expires = datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
c["ban"] = 1
c["ban"]["path"] = "/"
c["ban"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
try:
funct.logging('locahost', login+' failed log in', haproxywi=1, login=1)
except:
funct.logging('locahost', ' Failed log in. Wrong username', haproxywi=1)
print(c)
print("Content-type: text/html\n")
print('ban')
def get_cur():
try:
if mysql_enable == '0':
con = sqltool.connect(db, isolation_level=None)
else:
con = sqltool.connect(user=mysql_user, password=mysql_password,
host=mysql_host, port=mysql_port,
database=mysql_db)
cur = con.cursor()
except sqltool.Error as e:
funct.logging('DB ', ' '+e, haproxywi=1, login=1)
else:
return con, cur
def edit_section(server_id):
body = request.body.getvalue().decode('utf-8')
section_name = request.headers.get('section-name')
save = request.headers.get('action')
token = request.headers.get('token')
servers = check_permit_to_server(server_id)
hap_configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir')
login, group_id = sql.get_username_groupid_from_api_token(token)
if save == '':
save = 'save'
elif save == 'restart':
save = ''
elif save == 'reload':
save = 'reload'
for s in servers:
ip = s[2]
cfg = '/tmp/' + ip + '.cfg'
out = funct.get_config(ip, cfg)
start_line, end_line, config_read = funct.get_section_from_config(cfg, section_name)
returned_config = funct.rewrite_section(start_line, end_line, cfg, body)
try:
cfg_for_save = hap_configs_dir + ip + "-" + funct.get_data('config') + ".cfg"
try:
with open(cfg, "w") as conf:
conf.write(returned_config)
return_mess = 'section has been updated'
os.system("/bin/cp %s %s" % (cfg, cfg_for_save))
out = funct.master_slave_upload_and_restart(ip, cfg, save, login=login)
funct.logging('localhost', " section " + section_name + " has been edited via API", login=login)
funct.logging(ip, 'Section ' + section_name + ' has been edited via API', haproxywi=1, login=login,
keep_history=1, service='haproxy')
if out:
return_mess = out
except IOError:
return_mess = "cannot upload config"
data = {server_id: return_mess}
except Exception as e:
data = {server_id: {"error": str(e)}}
return dict(error=data)
return dict(config=data)
def main(serv, port):
port = str(port)
firstrun = True
currentstat = []
readstats = ""
killer = GracefulKiller()
while True:
try:
readstats = subprocess.check_output(["echo show stat | nc "+serv+" "+port], shell=True)
except:
print("Unexpected error:", sys.exc_info())
sys.exit()
vips = readstats.splitlines()
for i in range(0,len(vips)):
if "UP" in str(vips[i]):
currentstat.append("UP")
elif "DOWN" in str(vips[i]):
currentstat.append("DOWN")
elif "MAINT" in str(vips[i]):
currentstat.append("MAINT")
else:
currentstat.append("none")
if firstrun == False:
if (currentstat[i] != oldstat[i] and currentstat[i]!="none") and ("FRONTEND" not in str(vips[i]) and "BACKEND" not in str(vips[i])):
servername = str(vips[i])
servername = servername.split(",")
realserver = servername[0]
server = servername[1]
alert = "Backend: "+realserver[2:]+", server: "+server+" has changed status and is now "+ currentstat[i] + " at " + serv
funct.telegram_send_mess(str(alert), ip=serv)
funct.logging("localhost", " "+alert, alerting=1)
firstrun = False
oldstat = []
oldstat = currentstat
currentstat = []
time.sleep(60)
if killer.kill_now:
break
funct.logging("localhost", " Worker shutdown for: "+serv, alerting=1)
def add_to_config(id):
data = {}
body = request.body.getvalue().decode('utf-8')
save = request.headers.get('action')
hap_configs_dir = funct.get_config_var('configs',
'haproxy_save_configs_dir')
login = request.headers.get('login')
if save == '':
save = 'save'
elif save == 'restart':
save = ''
try:
servers = check_permit_to_server(id)
for s in servers:
ip = s[2]
cfg = '/tmp/' + ip + '.cfg'
cfg_for_save = hap_configs_dir + ip + "-" + funct.get_data(
'config') + ".cfg"
out = funct.get_config(ip, cfg)
try:
with open(cfg, "a") as conf:
conf.write('\n' + body + '\n')
return_mess = 'section was added to the config'
os.system("/bin/cp %s %s" % (cfg, cfg_for_save))
funct.logging('localhost',
" section was added via REST API",
login=login)
out = funct.upload_and_restart(ip, cfg, just_save=save)
if out:
return_mess = out
except IOError:
return_mess = "cannot upload config"
data = {id: return_mess}
except:
data = {}
data[id] = {"error": "Cannot find the server"}
return dict(error=data)
return dict(config=data)
def main():
sql.delete_mentrics()
sql.delete_waf_mentrics()
servers = sql.select_servers_metrics_for_master()
started_workers = get_worker()
servers_list = []
for serv in servers:
servers_list.append(serv[0])
need_kill=list(set(started_workers) - set(servers_list))
need_start=list(set(servers_list) - set(started_workers))
if need_kill:
for serv in need_kill:
kill_worker(serv)
if need_start:
for serv in need_start:
start_worker(serv)
try:
waf_servers = sql.select_all_waf_servers()
waf_started_workers = get_waf_worker()
waf_servers_list = []
for serv in waf_servers:
waf_servers_list.append(serv[0])
waf_need_kill=list(set(waf_started_workers) - set(waf_servers_list))
waf_need_start=list(set(waf_servers_list) - set(waf_started_workers))
if waf_need_kill:
for serv in waf_need_kill:
kill_waf_worker(serv)
if waf_need_start:
for serv in waf_need_start:
start_waf_worker(serv)
except Exception as e:
funct.logging("localhost", 'Problems with WAF worker metrics '+e, metrics=1)
pass
def check_token_exists(token):
try:
import http.cookies
import os
cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
user_id = cookie.get('uuid')
if get_token(user_id.value) == token:
return True
else:
try:
funct.logging('localhost', ' tried do action with wrong token', haproxywi=1, login=1)
except:
funct.logging('localhost', ' An action with wrong token', haproxywi=1)
return False
except:
try:
funct.logging('localhost', ' cannot check token', haproxywi=1, login=1)
except:
funct.logging('localhost', ' Cannot check token', haproxywi=1)
return False
print('<option value="%s" %s>%s</option>' %
(files, selected, files))
print('</select>')
print('<input type="hidden" value="%s" name="serv">' % serv)
print('<input type="hidden" value="open" name="open">')
print(
'<button type="submit" value="Select" name="Select">Select</button></form>'
)
Select = form.getvalue('Select')
if Select is not None:
configver = form.getvalue('configver')
funct.logging(serv, "open old config %s" % configver)
print("<h3>Config from %s, and version is: %s</h3>" %
(serv, configver))
print('<form action="configver.py#conf" method="get">')
print('<input type="hidden" value="%s" name="serv">' % serv)
print('<input type="hidden" value="%s" name="configver">' % configver)
print('<input type="hidden" value="1" name="config">')
print('<a name="conf"></a>')
print(
'<p class="accordion-expand-holder">'
'<a class="accordion-expand-all ui-button ui-widget ui-corner-all" href="#">Expand all</a>'
'</p></center>')
funct.show_config(configver)
print('<center><p>')
funct.get_button("Just save", value="save")
def kill_all_workers():
cmd = "ps ax |grep -e 'tools/metrics_worker.py\|tools/metrics_waf_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost",
" Masrer killing all metrics workers",
metrics=1)
if stderr:
funct.logging("localhost", stderr, metrics=1)
def get_worker():
cmd = "ps ax |grep 'tools/metrics_worker.py' |grep -v grep |awk '{print $7}'"
output, stderr = funct.subprocess_execute(cmd)
if stderr:
funct.logging("localhost", stderr, metrics=1)
return output
if __name__ == "__main__":
funct.logging("localhost", " Metrics master started", metrics=1)
killer = GracefulKiller()
while True:
main()
time.sleep(20)
if killer.kill_now:
break
kill_all_workers()
funct.logging("localhost", " Master shutdown", metrics=1)
action = "versions.py"
configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir')
format = 'cfg'
if serv is not None and form.getvalue('del') is not None:
if Select is not None:
aftersave = 1
env = Environment(loader=FileSystemLoader('templates/'))
template = env.get_template('delver.html')
for get in form:
if format in get:
try:
os.remove(os.path.join(configs_dir, form.getvalue(get)))
file.add(form.getvalue(get) + "<br />")
funct.logging(
serv, "versions.py were deleted configs: %s" %
form.getvalue(get))
except OSError as e:
stderr = "Error: %s - %s." % (e.filename, e.strerror)
print(
'<meta http-equiv="refresh" content="10; url=versions.py?service=%s&serv=%s&open=open">'
% (service, form.getvalue('serv')))
if serv is not None and form.getvalue('config') is not None:
configver = configs_dir + configver
save = form.getvalue('save')
aftersave = 1
try:
funct.logging(serv, "configver.py upload old config %s" % configver)
except:
pass
with open(ssh_keys, "w") as conf:
conf.write(form.getvalue('ssh_cert'))
except IOError:
print(
'<div class="alert alert-danger">Can\'t save ssh keys file. Check ssh keys path in config</div>'
)
else:
print(
'<div class="alert alert-success">Ssh key was save into: %s </div>'
% ssh_keys)
try:
cmd = 'chmod 600 %s' % ssh_keys
funct.subprocess_execute(cmd)
except IOError as e:
funct.logging('localhost', e.args[0], haproxywi=1)
try:
funct.logging("local",
"users.py#ssh upload new ssh cert %s" % ssh_keys)
except:
pass
if serv and form.getvalue('ssl_cert'):
cert_local_dir = funct.get_config_var('main', 'cert_local_dir')
cert_path = sql.get_setting('cert_path')
if not os.path.exists(cert_local_dir):
os.makedirs(cert_local_dir)
if form.getvalue('ssl_name') is None:
with open(ssh_keys, "w") as conf:
conf.write(form.getvalue('ssh_cert'))
except IOError:
print(
'<div class="alert alert-danger">Can\'t save ssh keys file. Check ssh keys path in config</div>'
)
else:
print(
'<div class="alert alert-success">Ssh key was save into: %s </div>'
% ssh_keys)
try:
cmd = 'chmod 600 %s' % ssh_keys
funct.subprocess_execute(cmd)
except IOError as e:
funct.logging('localhost', e.args[0], haproxywi=1)
try:
funct.logging("local",
"users.py#ssh upload new ssh cert %s" % ssh_keys)
except:
pass
if serv and form.getvalue('ssl_cert'):
cert_local_dir = funct.get_config_var('main', 'cert_local_dir')
cert_path = sql.get_setting('cert_path')
if not os.path.exists(cert_local_dir):
os.makedirs(cert_local_dir)
if form.getvalue('ssl_name') is None:
user_id = cookie.get('uuid')
user = sql.get_user_name_by_uuid(user_id.value)
servers = sql.get_dick_permit()
token = sql.get_token(user_id.value)
except:
pass
hap_configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir')
if serv is not None:
cfg = hap_configs_dir + serv + "-" + funct.get_data('config') + ".cfg"
if serv is not None and form.getvalue('open') is not None:
try:
funct.logging(serv, "config.py open config")
except:
pass
error = funct.get_config(serv, cfg)
try:
conf = open(cfg, "r")
config_read = conf.read()
conf.close
except IOError:
error += '<br />Can\'t read import config file'
os.system("/bin/mv %s %s.old" % (cfg, cfg))
if serv is not None and form.getvalue('config') is not None:
config_add = "\n" + name + new_userlist_groups + new_users_list
try:
if config_add:
hap_configs_dir = funct.get_config_var('configs',
'haproxy_save_configs_dir')
cfg = hap_configs_dir + serv + "-" + funct.get_data('config') + ".cfg"
funct.get_config(serv, cfg)
try:
with open(cfg, "a") as conf:
conf.write(config_add)
except IOError:
print("Can't read import config file")
funct.logging(serv, "add.py add new %s" % name)
print('<div class="line3">')
MASTERS = sql.is_master(serv)
for master in MASTERS:
if master[0] != None:
funct.upload_and_restart(master[0], cfg)
stderr = funct.upload_and_restart(serv, cfg, just_save="save")
if stderr:
print('<div class="alert alert-danger">%s</div>' % stderr)
else:
print(
'<meta http-equiv="refresh" content="0; url=add.py?add=%s&conf=%s&serv=%s">'
% (name, config_add, serv))
def kill_all_workers():
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost", " Masrer killing all workers", alerting=1)
if stderr:
funct.logging("localhost", stderr, alerting=1)
session_ttl = int(session_ttl)
expires = datetime.datetime.utcnow() + datetime.timedelta(days=session_ttl)
user_uuid = str(uuid.uuid4())
user_token = str(uuid.uuid4())
for users in USERS:
if login in users[1] and password == users[3]:
c = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
c["uuid"] = user_uuid
c["uuid"]["path"] = "/app/"
c["uuid"]["expires"] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
print(c)
sql.write_user_uuid(login, user_uuid)
sql.write_user_token(login, user_token)
funct.logging('locahost', sql.get_user_name_by_uuid(user_uuid)+' log in')
print("Content-type: text/html\n")
print('ok')
sys.exit()
print("Content-type: text/html\n")
print('<center><div class="alert alert-danger">Somthing wrong :( I\'m sad about this, but try again!</div><br /><br />')
sys.exit()
if login is None:
print("Content-type: text/html\n")
if create_db.check_db():
if create_db.create_table():
create_db.update_all()
db_create = '<div class="alert alert-success">DB was created<br /><br />Now you can login, default: admin/admin</div>'
name = form.getvalue('name')
ssh_keys = fullpath + '/keys/' + name + '.pem'
try:
with open(ssh_keys, "w") as conf:
conf.write(form.getvalue('ssh_cert'))
except IOError:
print(
'<div class="alert alert-danger">Can\'t save ssh keys file. Check ssh keys path in config</div>'
)
else:
print(
'<div class="alert alert-success">Ssh key was save into: %s </div>'
% ssh_keys)
try:
funct.logging("local",
"users.py#ssh upload new ssh cert %s" % ssh_keys)
except:
pass
if serv and form.getvalue('ssl_cert'):
cert_local_dir = funct.get_config_var('main', 'cert_local_dir')
cert_path = funct.get_config_var('haproxy', 'cert_path')
if form.getvalue('ssl_name') is None:
print(
'<div class="alert alert-danger">Please enter desired name</div>')
else:
name = form.getvalue('ssl_name') + '.pem'
try:
with open(name, "w") as ssl_cert:
if serv == ip[0]:
return_files.add(file)
return sorted(return_files, reverse=True)
if serv is not None and form.getvalue('open') is not None:
if Select is not None:
aftersave = 1
for get in form:
if "cfg" in get:
try:
os.remove(os.path.join(hap_configs_dir,
form.getvalue(get)))
file.add(form.getvalue(get) + "<br />")
funct.logging(
serv,
"delver.py deleted config: %s" % form.getvalue(get))
except OSError:
stderr = "Error: %s - %s." % (e.filename, e.strerror)
print(
'<meta http-equiv="refresh" content="10; url=delver.py?serv=%s&open=open">'
% form.getvalue('serv'))
output_from_parsed_template = template.render(
h2=1,
title="Delete old versions HAProxy config",
role=sql.get_user_role_by_uuid(user_id.value),
action="delver.py",
user=user,
select_id="serv",
serv=serv,
file_format = 'cfg'
servers = sql.get_dick_permit(haproxy=1)
if serv is not None:
if service == 'nginx':
conf_file_name_short = config_file_name.split('/')[-1]
cfg = configs_dir + serv + "-" + conf_file_name_short + "-" + funct.get_data('config') + "." + file_format
else:
cfg = configs_dir + serv + "-" + funct.get_data('config') + "."+file_format
if serv is not None and form.getvalue('open') is not None and form.getvalue('new_config') is None:
funct.check_is_server_in_group(serv)
if service == 'keepalived':
error = funct.get_config(serv, cfg, keepalived=1)
try:
funct.logging(serv, " Keepalived config has been opened for ")
except Exception:
pass
elif service == 'nginx':
error = funct.get_config(serv, cfg, nginx=1, config_file_name=config_file_name)
try:
funct.logging(serv, " Nginx config has been opened ")
except Exception:
pass
else:
error = funct.get_config(serv, cfg)
try:
funct.logging(serv, " HAProxy config has been opened ")
except Exception:
pass
def get_worker():
cmd = "ps ax |grep 'tools/metrics_worker.py' |grep -v grep |awk '{print $7}'"
output, stderr = funct.subprocess_execute(cmd)
if stderr:
funct.logging("localhost", stderr, metrics=1)
return output
def kill_all_workers():
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost", " Masrer killing all workers", alerting=1)
if stderr:
funct.logging("localhost", stderr, alerting=1)
def get_worker():
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $7}'"
output, stderr = funct.subprocess_execute(cmd)
if stderr:
funct.logging("localhost", stderr, alerting=1)
return output
if __name__ == "__main__":
funct.logging("localhost", " Master started", alerting=1)
killer = GracefulKiller()
while True:
main()
time.sleep(20)
if killer.kill_now:
break
kill_all_workers()
funct.logging("localhost", " Master shutdown", alerting=1)
