def check_log_system(printRes=False):
global results
logme.loader()
FWDIR = func.get_path("FWDIR")
CPDIR = func.get_path("CPDIR")
title = "Checking logs"
#
# Format: [file, search, exclude]
#
logfiles = [["/var/log/messages*", "fail|error", "xpand|failover"],
[CPDIR + "/log/cpd.elg", "fail|error", "PROVIDER-1|PA_status"]]
if func.isFirewall():
logfiles.append(
["/var/log/routed.log", "fail|error", "xpand|failover"])
logfiles.append([FWDIR + "/log/fwd.elg", "failed", "discntd"])
if func.isManagement():
logfiles.append([FWDIR + "/log/fwm.elg", "failed", "none"])
for log in logfiles:
logme.loader()
out, err = func.execute_command('cat ' + log[0] + ' | grep -viE "(' +
log[2] + ')" | grep -icE "(' + log[1] +
')"')
out = out.read().strip('\n')
state = "PASS"
detail = ""
if out != "0":
state = "FAIL"
detail = out + " messages"
results.append(
[title + " (" + log[0] + ")", detail, state, "Log Files"])
if printRes:
print_results()
def check_all(printRes=False, ftype="all"):
global results
title = "File"
files_arr = []
if ftype == "gw":
files_arr = files_fwd
if ftype == "mgmt":
files_arr = files_fwm
if ftype == "all":
if func.isFirewall():
files_arr = files_arr + files_fwd
if func.isManagement():
files_arr = files_arr + files_fwm
i = 0
while i < len(files_arr):
state = "PASS"
detail = ""
try:
with open(files_arr[i][0], "rb") as f:
bytes = f.read()
fhash = hashlib.md5(bytes).hexdigest()
if fhash != files_arr[i][1]:
state = "WARN"
detail = "Wrong Hash!"
except:
if files_arr[i][1] != "1":
state = "FAIL"
detail = "not found!"
results.append([
title + " (" + files_arr[i][0] + ")", detail, state, "Filesystem"
])
i = i + 1
if printRes:
logme.results(results)
results = []
def check_cpumemif(printRes = False): global results if func.isFirewall(): health.check_failedalloc() health.check_cpu() health.check_memory() health.check_interfaces() results = health.get_results(True) if printRes: print_results()
def check_all(printRes=False):
check_diskspace()
check_cpu()
check_memory()
check_crash()
check_log_system()
check_process()
check_ntp()
if func.isFirewall():
check_fw_fragments()
check_fw_aggressive()
check_table_overflow()
check_failedalloc()
check_interfaces()
check_sic_state()
check_overlap_encdom()
check_licensing()
check_blade_update()
check_dispatcher()
check_multik_stat()
check_blades()
check_parsers()
if func.isFirewall() and func.isCluster():
check_clusterxl_state()
check_clusterxl_sync()
check_clusterxl_pnote()
check_fwha_version()
if func.fwVersion() == "R80.30" or func.fwVersion() == "R80.40":
check_clusterxl_ccp()
if func.fwVersion() == "R80.40":
check_clusterxl_release()
if func.isManagement():
check_mgmt_status()
check_mgmt_api()
check_mgmt_gui()
check_mgmt_dblock()
check_mgmt_validations()
check_mgmt_updateips()
if len(func.ipmiInfo()) > 0:
check_ipmi_sensor()
if printRes:
print_results()
def html_intro(res):
html = ""
html += "<div class='container jumbotron'> \n"
html += html_intro_item_cli("Hostname", "hostname")
html += html_intro_item_cli("CP Version", "fw ver")
html += html_intro_item_cli("Kernel", "uname -a")
if func.isFirewall():
html += html_intro_item_cli("Policy", "fw stat | grep -v 'POLICY' | awk '{ print $2 }'")
html += html_intro_item_cli("Blades", "enabled_blades")
html += html_intro_item_cli("Uptime", "uptime")
html += html_intro_item("CPme Version", version + " (by Simon Brecht, https://github.com/0x7c2/cpme/)")
html += html_intro_item_cli("Created", "date")
html += html_intro_item("Status", html_badge_out(res))
html += "</div> \n"
return html
def check_all(printRes = False, runAll = False): global results if runAll: check_cpumemif() if func.isFirewall(): check_securexl() check_modules() check_multiq() check_prioq() check_ispredundancy() if runAll: files.check_all(False, "all") results = results + files.get_results(True) if printRes: print_results()
def __init__(self, debugLevel=0):
self.debugLevel = debugLevel
self.enabledBlades = func.enabledBlades()
self.modules = map(__import__, self.dynamic_load)
i = 0
for module in self.modules:
for name, obj in inspect.getmembers(module):
if inspect.isclass(obj):
if "check_" in name or "diag_" in name:
class_ = getattr(module, name)
self.classes[name] = class_(
func.fwVersion(),
isFw=func.isFirewall(),
isMgmt=func.isManagement(),
isCluster=func.isCluster(),
enabledBlades=self.enabledBlades,
debugLevel=self.debugLevel)
i = i + 1
self.run_self()
["Run zdebug with options", "troubleshooting.zdebug()"],
["Print connection table - raw", "troubleshooting.print_table('connections')"],
["Print connection table - formatted", "troubleshooting.print_table('connections', True)"],
["Clear connection table (ALL!)", "troubleshooting.clear_table('connections')"],
["Clear specific connections from table","troubleshooting.clear_table_input('connections')"],
["STOP CheckPoint Services", "troubleshooting.run_cpstop()"],
["STOP CheckPoint Services and keep policy","troubleshooting.run_cpstop('-fwflag -proc')"],
["UNLOAD Security/TP Policy", "troubleshooting.load_policy(False)"],
["FETCH Security/TP Policy", "troubleshooting.load_policy(True)"],
["Disable Antispoofing", "troubleshooting.run_spoofing(0)"],
["Enable Antispoofing", "troubleshooting.run_spoofing(1)"],
["ClusterXL Status", "troubleshooting.clusterxl_status()"],
["SecureXL DoS Mitigation Status", "troubleshooting.run_securexl_dos()"],
["Display VPN Tunnel Status", "troubleshooting.print_vpn()"]]
if func.isFirewall() and not func.isFWUserMode():
menu_item.append(["TOP 15 heavy F2F Connections (specific worker)", "troubleshooting.select_f2f_stats()"])
menu_item.append(["TOP 15 heavy F2F Connections (all worker!)", "troubleshooting.print_f2f_stats(-1)"])
if func.isFirewall() and func.isFWUserMode():
menu_item.append(["Display user-mode cpu ressources", "troubleshooting.run_top('-H -p `pidof fwk`')"])
if func.isFirewall():
menu_item.append(["Measure kernel delay (EXPERIMENTAL!)", "troubleshooting.fwkern_delay()"])
menu_item.append(["Disable IPS on the fly", "troubleshooting.run_ips(False)"])
menu_item.append(["Enable IPS on the fly", "troubleshooting.run_ips(True)"])
menu_item.append(["Print heavy conns detected by CoreXL", "troubleshooting.print_heavy_conn()"])
menu_item.append(["Back to Main Menu", "menu_set('main')"])
def check_cpu(printRes=False):
global results
title = "Checking CPU usage"
logme.loader()
if func.isFirewall():
out, err = func.execute_command("fw ctl affinity -l")
affinity = out.read()
else:
affinity = ""
dbcur = func.execute_sqlite_query(
"select name_of_cpu,max(cpu_usage) from UM_STAT_UM_CPU_UM_CPU_ORDERED_TABLE group by name_of_cpu;"
)
for row in dbcur:
worker = ""
nic = ""
daemon = ""
logme.loader()
cpu = row[0]
for line in affinity.split('\n'):
logme.loader()
if "CPU " + str(cpu) + '#' in line + '#':
if "Kernel" in line:
if worker != "":
worker = worker + ", "
worker = worker + line.split(":")[0].replace("Kernel ", "")
elif "Daemon" in line:
daemon = "Daemon(s), "
else:
if nic != "":
nic = nic + ", "
nic = nic + line.split(":")[0]
load = str(row[1]).split(".")[0]
state = "PASS"
if int(load) > 85 and nic != "":
state = "FAIL"
elif int(load) > 85 and nic == "":
state = "WARN"
if nic != "":
nic = nic + ", "
results.append([
title + " (peak - CPU " + str(cpu) + "): " + daemon + nic + worker,
load + "%", state, "CPU"
])
dbcur = func.execute_sqlite_query(
"select name_of_cpu,avg(cpu_usage) from UM_STAT_UM_CPU_UM_CPU_ORDERED_TABLE group by name_of_cpu;"
)
for row in dbcur:
worker = ""
nic = ""
daemon = ""
logme.loader()
cpu = row[0]
for line in affinity.split('\n'):
logme.loader()
if "CPU " + str(cpu) + '#' in line + '#':
if "Kernel" in line:
if worker != "":
worker = worker + ", "
worker = worker + line.split(":")[0].replace("Kernel ", "")
elif "Daemon" in line:
daemon = "Daemon(s), "
else:
if nic != "":
nic = nic + ", "
nic = nic + line.split(":")[0]
load = str(row[1]).split(".")[0]
state = "PASS"
if int(load) > 50:
state = "WARN"
if int(load) > 50 and nic != "":
state = "FAIL"
if int(load) > 85 and worker != "":
state = "FAIL"
if nic != "":
nic = nic + ", "
results.append([
title + " (avg - CPU " + str(cpu) + "): " + daemon + nic + worker,
load + "%", state, "CPU"
])
dbcur.close()
if printRes:
print_results()
import logme
import files
import func
import kernel
menu_text = "Health Analysis"
menu_item = [["Run all checks", "health.check_all(True)"],
["Check memory usage", "health.check_memory(True)"],
["Check disk space", "health.check_diskspace(True)"],
["Check cpu usage", "health.check_cpu(True)"],
["Check system crash", "health.check_crash(True)"],
["Check system logfiles", "health.check_log_system(True)"],
["Check processes", "health.check_process(True)"],
["Check NTP and Time", "health.check_ntp(True)"]]
if func.isFirewall():
menu_item.append(
["Check Firewall Fragments", "health.check_fw_fragments(True)"])
menu_item.append(
["Check kernel table overflow", "health.check_table_overflow(True)"])
menu_item.append(
["Check Aggressive Aging", "health.check_fw_aggressive(True)"])
menu_item.append(
["Check memory allocations", "health.check_failedalloc(True)"])
menu_item.append(
["Check interface statistics", "health.check_interfaces(True)"])
menu_item.append(["Check licensing", "health.check_licensing(True)"])
menu_item.append(
["Check overlapping encdoms", "health.check_overlap_encdom(True)"])
menu_item.append(["Check SIC State", "health.check_sic_state(True)"])
menu_item.append(["Check firewall mode", "health.check_fw_mode(True)"])
import rulebase
import gaia
menu_cur = "main"
menu_wait = True
menu_text = {}
menu_item = {}
menu_exec = {}
menu_text["main"] = "Main Menu"
menu_item["main"] = [["GAiA Operating System", "menu_set('gaia')"],
["Health Analysis", "menu_set('health')"],
["Files Analysis", "menu_set('files')"],
["Performance Analysis", "menu_set('performance')"]]
if func.isFirewall():
menu_item["main"].append(["Kernel Parameters", "menu_set('kernel')"])
menu_item["main"].append(["Tuning Options", "menu_set('tuning')"])
menu_item["main"].append(
["Troubleshooting Options", "menu_set('troubleshooting')"])
if func.isManagement():
menu_item["main"].append(["Management Options", "menu_set('mgmt')"])
menu_item["main"].append(
["Manage/Optimize Rulebase", "menu_set('rulebase')"])
menu_item["main"].append(["Create HTML Report", "func.make_report_html()"])
menu_text["gaia"] = gaia.add_text()
menu_item["gaia"] = gaia.add_item()
def run_check(self):
if func.isFirewall():
out, err = func.execute_command("fw ctl affinity -l")
affinity = out.read()
else:
affinity = ""
dbcur = func.execute_sqlite_query(
"select name_of_cpu,max(cpu_usage) from UM_STAT_UM_CPU_UM_CPU_ORDERED_TABLE group by name_of_cpu;"
)
for row in dbcur:
worker = ""
nic = ""
daemon = ""
cpu = row[0]
for line in affinity.split('\n'):
if "CPU " + str(cpu) + '#' in line + '#':
if "Kernel" in line:
if worker != "":
worker = worker + ", "
worker = worker + line.split(":")[0].replace(
"Kernel ", "")
elif "Daemon" in line:
daemon = "Daemon(s), "
else:
if nic != "":
nic = nic + ", "
nic = nic + line.split(":")[0]
load = str(row[1]).split(".")[0]
state = "PASS"
if int(load) > 85 and nic != "":
state = "FAIL"
elif int(load) > 85 and nic == "":
state = "WARN"
if nic != "":
nic = nic + ", "
self.add_result(
self.title + " (peak - CPU " + str(cpu) + "): " + daemon +
nic + worker, state, load + "%")
dbcur = func.execute_sqlite_query(
"select name_of_cpu,avg(cpu_usage) from UM_STAT_UM_CPU_UM_CPU_ORDERED_TABLE group by name_of_cpu;"
)
for row in dbcur:
worker = ""
nic = ""
daemon = ""
cpu = row[0]
for line in affinity.split('\n'):
if "CPU " + str(cpu) + '#' in line + '#':
if "Kernel" in line:
if worker != "":
worker = worker + ", "
worker = worker + line.split(":")[0].replace(
"Kernel ", "")
elif "Daemon" in line:
daemon = "Daemon(s), "
else:
if nic != "":
nic = nic + ", "
nic = nic + line.split(":")[0]
load = str(row[1]).split(".")[0]
state = "PASS"
if int(load) > 50:
state = "WARN"
if int(load) > 50 and nic != "":
state = "FAIL"
if int(load) > 85 and worker != "":
state = "FAIL"
if nic != "":
nic = nic + ", "
self.add_result(
self.title + " (avg - CPU " + str(cpu) + "): " + daemon + nic +
worker, state, load + "%")
dbcur.close()