def kill_waf_worker(serv):
cmd = "ps ax |grep 'tools/metrics_waf_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost", " Masrer killed WAF metrics worker for: "+serv, metrics=1)
if stderr:
funct.logging("localhost", stderr, metrics=1)
print(template)
id = form.getvalue('id')
name = form.getvalue('name')
ioloop = asyncio.get_event_loop()
ioloop.run_until_complete(get_runner_overviewServers(server1=name, server2=serv, id=id))
ioloop.close()
if act == "overviewHapwi":
from jinja2 import Environment, FileSystemLoader
env = Environment(loader=FileSystemLoader('templates/ajax'), autoescape=True)
template = env.get_template('/overviewHapwi.html')
cmd = "top -b -n 1 |head -12"
server_status, stderr = funct.subprocess_execute(cmd)
template = template.render(server_status=server_status,stderr=stderr)
print(template)
if form.getvalue('action'):
import requests
haproxy_user = sql.get_setting('stats_user')
haproxy_pass = sql.get_setting('stats_password')
stats_port = sql.get_setting('stats_port')
stats_page = sql.get_setting('stats_page')
postdata = {
'action' : form.getvalue('action'),
print('Content-type: text/html\n')
print(error_mess)
else:
import funct
print('Content-type: text/html\n')
fullpath = funct.get_config_var('main', 'fullpath')
for sshs in select_ssh(id=id):
ssh_enable = sshs[2]
ssh_key_name = fullpath + '/keys/%s.pem' % sshs[1]
new_ssh_key_name = fullpath + '/keys/%s.pem' % name
if ssh_enable == 1:
cmd = 'mv %s %s' % (ssh_key_name, new_ssh_key_name)
try:
funct.subprocess_execute(cmd)
except:
pass
update_ssh(id, name, enable, group, username, password)
if form.getvalue('new_ssh'):
name = form.getvalue('new_ssh')
enable = form.getvalue('ssh_enable')
group = form.getvalue('new_group')
username = form.getvalue('ssh_user')
password = form.getvalue('ssh_pass')
page = form.getvalue('page')
page = page.split("#")[0]
if username is None or name is None:
print('Content-type: text/html\n')
print(error_mess)
print('Content-type: text/html\n')
funct.check_login()
try:
user, user_id, role, token, servers, user_services = funct.get_users_params(
)
services = []
except:
pass
form = funct.form
serv = funct.is_ip_or_dns(form.getvalue('serv'))
service = form.getvalue('service')
autorefresh = 0
cmd = "ps ax |grep -e 'keep_alive.py' |grep -v grep |wc -l"
keep_alive, stderr = funct.subprocess_execute(cmd)
if service == 'nginx':
if funct.check_login(service=2):
title = 'Nginx servers overview'
if serv:
if funct.check_is_server_in_group(serv):
servers = sql.select_servers(server=serv)
autorefresh = 1
server_id = sql.select_server_id_by_ip(serv)
service_settings = sql.select_docker_service_settings(
server_id, service)
else:
servers = sql.get_dick_permit(virt=1, nginx=1)
service_settings = sql.select_docker_services_settings(service)
elif service == 'keepalived':
import funct
import sql
from jinja2 import Environment, FileSystemLoader
env = Environment(extensions=["jinja2.ext.do"],
loader=FileSystemLoader('templates/'),
autoescape=True)
template = env.get_template('servers.html')
form = funct.form
print('Content-type: text/html\n')
funct.check_login()
funct.page_for_admin(level=2)
try:
user, user_id, role, token, servers = funct.get_users_params()
ldap_enable = sql.get_setting('ldap_enable')
grafana, stderr = funct.subprocess_execute(
"service grafana-server status |grep Active |awk '{print $1}'")
user_group = funct.get_user_group(id=1)
settings = sql.get_setting('', all=1)
geoip_country_codes = sql.select_geoip_country_codes()
services = []
services_name = {
'checker_haproxy': 'Checker backends master service',
}
for s, v in services_name.items():
cmd = "systemctl status %s |grep Act |awk '{print $2}'" % s
status, stderr = funct.subprocess_execute(cmd)
if s != 'keep_alive':
service_name = s.split('_')[0]
else:
service_name = s
minut1 = form.getvalue('minut1')
date = hour+':'+minut
date1 = hour1+':'+minut1
if grep is not None:
grep_act = '|grep'
else:
grep_act = ''
grep = ''
if serv == 'haproxy-wi.access.log':
cmd="cat %s| awk -F\"/|:\" '$3>\"%s:00\" && $3<\"%s:00\"' |tail -%s %s %s" % ('/var/log/httpd/'+serv, date, date1, rows, grep_act, grep)
else:
cmd="cat %s| awk '$4>\"%s:00\" && $4<\"%s:00\"' |tail -%s %s %s" % ('/var/log/httpd/'+serv, date, date1, rows, grep_act, grep)
output, stderr = funct.subprocess_execute(cmd)
funct.show_log(output)
print(stderr)
if form.getvalue('viewlogs') is not None:
viewlog = form.getvalue('viewlogs')
log_path = funct.get_config_var('main', 'log_path')
rows = form.getvalue('rows2')
grep = form.getvalue('grep')
hour = form.getvalue('hour')
minut = form.getvalue('minut')
hour1 = form.getvalue('hour1')
minut1 = form.getvalue('minut1')
date = hour+':'+minut
date1 = hour1+':'+minut1
def kill_all_workers():
cmd = "ps ax |grep 'tools/checker_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost", " Master killing all workers", alerting=1)
if stderr:
funct.logging("localhost", stderr, alerting=1)
service = form.getvalue('service')
autorefresh = 0
if service == 'nginx':
title = "Nginx servers overview"
keep_alive = ''
stderr = ''
servers = sql.get_dick_permit(virt=1, nginx=1)
service = 'nginx'
if serv:
servers = sql.select_servers(server=serv)
autorefresh = 1
else:
title = "HAProxy servers overview"
cmd = "ps ax |grep -e 'keep_alive.py' |grep -v grep |wc -l"
keep_alive, stderr = funct.subprocess_execute(cmd)
service = 'haproxy'
if serv:
servers = sql.select_servers(server=serv)
autorefresh = 1
else:
servers = sql.get_dick_permit(virt=1, haproxy=1)
haproxy_sock_port = sql.get_setting('haproxy_sock_port')
servers_with_status1 = []
out1 = ''
for s in servers:
servers_with_status = list()
servers_with_status.append(s[0])
servers_with_status.append(s[1])
servers_with_status.append(s[2])
print('Content-type: text/html\n')
if create_db.check_db():
if create_db.create_table():
create_db.update_all()
create_db.update_all_silent()
funct.check_login()
try:
cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
user_id = cookie.get('uuid')
user = sql.get_user_name_by_uuid(user_id.value)
users = sql.select_users()
groups = sql.select_groups()
token = sql.get_token(user_id.value)
cmd = "ps ax |grep checker_mas |grep -v grep |wc -l"
checker_master, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep checker_worker |grep -v grep |wc -l"
checker_worker, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep metrics_master |grep -v grep |wc -l"
metrics_master, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep -e 'metrics_worker\|metrics_waf_worker.py' |grep -v grep |wc -l"
metrics_worker, stderr = funct.subprocess_execute(cmd)
except:
pass
template = template.render(h2=1,
autorefresh=1,
title="Overview",
role=sql.get_user_role_by_uuid(user_id.value),
user=user,
users=users,
print('Content-type: text/html\n')
if create_db.check_db():
if create_db.create_table():
create_db.update_all()
create_db.update_all_silent()
funct.check_login()
try:
cookie = http.cookies.SimpleCookie(os.environ.get("HTTP_COOKIE"))
user_id = cookie.get('uuid')
user = sql.get_user_name_by_uuid(user_id.value)
users = sql.select_users()
groups = sql.select_groups()
token = sql.get_token(user_id.value)
cmd = "ps ax |grep checker_mas |grep -v grep |wc -l"
checker_master, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep checker_worker |grep -v grep |wc -l"
checker_worker, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep metrics_master |grep -v grep |wc -l"
metrics_master, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep -e 'metrics_worker\|metrics_waf_worker.py' |grep -v grep |wc -l"
metrics_worker, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep -e 'keep_alive.py' |grep -v grep |wc -l"
keep_alive, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep '(wsgi:api)'|grep -v grep|wc -l"
api, stderr = funct.subprocess_execute(cmd)
except:
pass
template = template.render(h2=1,
autorefresh=1,
#!/usr/bin/env python3
import funct
import sql
from jinja2 import Environment, FileSystemLoader
env = Environment(loader=FileSystemLoader('templates/'), autoescape=True)
template = env.get_template('metrics.html')
print('Content-type: text/html\n')
funct.check_login()
try:
user, user_id, role, token, servers = funct.get_users_params()
cmd = "rpm --query haproxy-wi-metrics-* |awk -F\"metrics\" '{print $2}' |awk -F\".noa\" '{print $1}' |sed 's/-//1' |sed 's/-/./'"
service_ver, stderr = funct.subprocess_execute(cmd)
services = '0'
if not stderr:
if service_ver[0] == '* is not installed':
servers = ''
else:
servers = sql.select_servers_metrics(user_id.value)
services = '1'
except Exception:
pass
template = template.render(h2=1,
title="Metrics",
autorefresh=1,
role=role,
user=user,
users = sql.select_users()
settings = sql.get_setting('', all=1)
ldap_enable = sql.get_setting('ldap_enable')
services = []
services_name = {
"checker_haproxy": "Master checker service",
"keep_alive": "Auto start service",
"metrics_haproxy": "Master metrics service",
"prometheus": "Prometheus service",
"grafana-server": "Grafana service",
"smon": "Simple monitoring network ports",
"fail2ban": "Fail2ban service"
}
for s, v in services_name.items():
cmd = "systemctl status %s |grep Act |awk '{print $2}'" % s
status, stderr = funct.subprocess_execute(cmd)
services.append([s, status, v])
except:
pass
template = template.render(title="Admin area: Manage users",
role=role,
user=user,
users=users,
groups=sql.select_groups(),
servers=sql.select_servers(full=1),
roles=sql.select_roles(),
masters=sql.select_servers(get_master_servers=1),
sshs=sql.select_ssh(),
telegrams=sql.select_telegram(),
token=token,
from jinja2 import Environment, FileSystemLoader
env = Environment(loader=FileSystemLoader('templates/'), autoescape=True)
template = env.get_template('admin.html')
form = funct.form
print('Content-type: text/html\n')
funct.check_login()
funct.page_for_admin()
try:
user, user_id, role, token, servers = funct.get_users_params()
users = sql.select_users()
settings = sql.get_setting('', all=1)
ldap_enable = sql.get_setting('ldap_enable')
grafana, stderr = funct.subprocess_execute("service grafana-server status |grep Active |awk '{print $1}'")
services = []
services_name = {'checker_haproxy': 'Master backends checker service',
'keep_alive': 'Auto start service',
'metrics_haproxy': 'Master metrics service',
'prometheus': 'Prometheus service',
'grafana-server': 'Grafana service',
'smon': 'Simple monitoring network ports',
'fail2ban': 'Fail2ban service'}
for s, v in services_name.items():
cmd = "systemctl status %s |grep Act |awk '{print $2}'" % s
status, stderr = funct.subprocess_execute(cmd)
if s != 'keep_alive':
service_name = s.split('_')[0]
else:
print('Content-type: text/html\n')
funct.check_login()
funct.page_for_admin()
log_path = funct.get_config_var('main', 'log_path')
time_storage = sql.get_setting('log_time_storage')
time_storage = int(time_storage)
try:
time_storage_hours = time_storage * 24
for dirpath, dirnames, filenames in os.walk(log_path):
for file in filenames:
curpath = os.path.join(dirpath, file)
try:
funct.subprocess_execute('sudo chown apache:apache ' + curpath)
except:
pass
file_modified = datetime.datetime.fromtimestamp(
os.path.getmtime(curpath))
if datetime.datetime.now() - file_modified > datetime.timedelta(
hours=time_storage_hours):
os.remove(curpath)
except:
print(
'<center><div class="alert alert-danger" style="margin: 0; margin-bottom: 10px;">Can\'t delete old logs file. <br> Please check "log_time_storage" in config and <br>exist directory </div></center>'
)
pass
try:
user, user_id, role, token, servers = funct.get_users_params()
def kill_all_workers():
cmd = "ps ax |grep -e 'tools/metrics_worker.py\|tools/metrics_waf_worker.py' |grep -v grep |awk '{print $1}' |xargs kill"
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost", " Masrer killing all metrics workers", metrics=1)
if stderr:
funct.logging("localhost", stderr, metrics=1)
if (role == 2 or role == 3) and int(user_group) != 1:
users = sql.select_users(group=user_group)
servers_for_grep = ''
i = 0
servers_len = len(servers)
for s in servers:
i += 1
if i != servers_len:
servers_for_grep += s[2] + '|'
else:
servers_for_grep += s[2]
cmd = "ps ax |grep -e 'metrics_worker\|metrics_waf_worker.py'|grep -E %s|grep -v grep |wc -l" % servers_for_grep
metrics_worker, stderr = funct.subprocess_execute(cmd)
cmd = "ps ax |grep checker_worker|grep -E %s |grep -v grep |wc -l" % servers_for_grep
checker_worker, stderr = funct.subprocess_execute(cmd)
i = 0
for s in sql.select_alert(group=user_group):
i += 1
is_checker_worker = i
is_metrics_workers = sql.select_servers_metrics_for_master(
group=user_group)
i = 0
for s in is_metrics_workers:
i += 1
is_metrics_worker = i
grafana = ''
prometheus = ''
host = ''
def get_worker():
cmd = "ps ax |grep 'tools/metrics_worker.py' |grep -v grep |awk '{print $7}'"
output, stderr = funct.subprocess_execute(cmd)
if stderr:
funct.logging("localhost", stderr, metrics=1)
return output
if act == "overviewServers":
id = form.getvalue('id')
name = form.getvalue('name')
ioloop = asyncio.get_event_loop()
ioloop.run_until_complete(
get_runner_overviewServers(server1=name, server2=serv, id=id))
ioloop.close()
if act == "overviewHapwi":
from jinja2 import Environment, FileSystemLoader
env = Environment(loader=FileSystemLoader('templates/ajax'),
autoescape=True)
template = env.get_template('/overviewHapwi.html')
cmd = "top -b -n 1 |head -9"
server_status, stderr = funct.subprocess_execute(cmd)
template = template.render(server_status=server_status, stderr=stderr)
print(template)
if form.getvalue('action'):
import requests
from requests_toolbelt.utils import dump
haproxy_user = sql.get_setting('stats_user')
haproxy_pass = sql.get_setting('stats_password')
stats_port = sql.get_setting('stats_port')
stats_page = sql.get_setting('stats_page')
postdata = {
'action': form.getvalue('action'),
def get_map(serv):
from datetime import datetime
from pytz import timezone
import networkx as nx
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
stats_port= sql.get_setting('stats_port')
haproxy_config_path = sql.get_setting('haproxy_config_path')
hap_configs_dir = funct.get_config_var('configs', 'haproxy_save_configs_dir')
date = funct.get_data('config')
cfg = hap_configs_dir + serv + "-" + date + ".cfg"
print('<center>')
print("<h3>Map from %s</h3><br />" % serv)
G = nx.DiGraph()
funct.get_config(serv, cfg)
try:
conf = open(cfg, "r")
except IOError:
print('<div class="alert alert-danger">Can\'t read import config file</div>')
node = ""
line_new2 = [1,""]
i = 1200
k = 1200
j = 0
m = 0
for line in conf:
if "listen" in line or "frontend" in line:
if "stats" not in line:
node = line
i = i - 500
if line.find("backend") == 0:
node = line
i = i - 500
G.add_node(node,pos=(k,i),label_pos=(k,i+150))
if "bind" in line:
try:
bind = line.split(":")
if stats_port not in bind[1]:
bind[1] = bind[1].strip(' ')
bind = bind[1].split("crt")
node = node.strip(' \t\n\r')
node = node + ":" + bind[0]
G.add_node(node,pos=(k,i),label_pos=(k,i+150))
except:
pass
if "server " in line or "use_backend" in line or "default_backend" in line and "stats" not in line:
if "timeout" not in line and "default-server" not in line and "#" not in line and "stats" not in line:
i = i - 300
j = j + 1
if "check" in line:
line_new = line.split("check")
else:
line_new = line.split("if ")
if "server" in line:
line_new1 = line_new[0].split("server")
line_new[0] = line_new1[1]
line_new2 = line_new[0].split(":")
line_new[0] = line_new2[0]
line_new[0] = line_new[0].strip(' \t\n\r')
line_new2[1] = line_new2[1].strip(' \t\n\r')
if j % 2 == 0:
G.add_node(line_new[0],pos=(k+250,i-350),label_pos=(k+225,i-100))
else:
G.add_node(line_new[0],pos=(k-250,i-50),label_pos=(k-225,i+180))
if line_new2[1] != "":
G.add_edge(node, line_new[0], port=line_new2[1])
else:
G.add_edge(node,line_new[0])
os.system("/bin/rm -f " + cfg)
pos=nx.get_node_attributes(G,'pos')
pos_label=nx.get_node_attributes(G,'label_pos')
edge_labels = nx.get_edge_attributes(G,'port')
try:
plt.figure(10,figsize=(9.5,15))
nx.draw(G, pos, with_labels=False, font_weight='bold', width=3, alpha=0.1,linewidths=5)
nx.draw_networkx_nodes(G,pos, node_color="skyblue", node_size=100, alpha=0.8, node_shape="p")
nx.draw_networkx_labels(G,pos=pos_label, alpha=1, font_color="green", font_size=10)
nx.draw_networkx_edges(G,pos, width=0.5,alpha=0.5, edge_color="#5D9CEB",arrows=False)
nx.draw_networkx_edge_labels(G, pos,label_pos=0.5,font_color="blue", labels=edge_labels, font_size=8)
plt.savefig("map.png")
plt.show()
except Exception as e:
print('<div class="alert alert-danger">' + str(e) + '</div>')
cmd = "rm -f "+os.path.dirname(os.getcwd())+"/map*.png && mv map.png "+os.path.dirname(os.getcwd())+"/map"+date+".png"
output, stderr = funct.subprocess_execute(cmd)
print(stderr)
print('<img src="/map%s.png" alt="map">' % date)
def kill_worker(serv):
cmd = "ps ax |grep 'tools/checker_worker.py %s'|grep -v grep |awk '{print $1}' |xargs kill" % serv
output, stderr = funct.subprocess_execute(cmd)
funct.logging("localhost", " Master killed worker for: "+serv, alerting=1)
if stderr:
funct.logging("localhost", stderr, alerting=1)
