def collect_perms_state():
out_string = ""
for check_dir in [
constants.PCMK_LIB, constants.PE_STATE_DIR, constants.CIB_DIR
]:
flag = 0
out_string += "##### Check perms for %s: " % check_dir
stat_info = os.stat(check_dir)
if not stat.S_ISDIR(stat_info.st_mode):
flag = 1
out_string += "\n%s wrong type or doesn't exist\n" % check_dir
continue
if stat_info.st_uid != pwd.getpwnam('hacluster')[2] or\
stat_info.st_gid != pwd.getpwnam('hacluster')[3] or\
"%04o" % (stat_info.st_mode & 0o7777) != "0750":
flag = 1
out_string += "\nwrong permissions or ownership for %s: " % check_dir
out_string += utillib.get_command_info(
"ls -ld %s" % check_dir)[1] + '\n'
if flag == 0:
out_string += "OK\n"
perms_f = os.path.join(constants.WORKDIR, constants.PERMISSIONS_F)
crmutils.str2file(out_string, perms_f)
def collect_crm_config():
workdir = constants.WORKDIR
if os.path.isfile(os.path.join(workdir, constants.CIB_F)):
cmd = r"CIB_file=%s/%s crm configure show" % (workdir, constants.CIB_F)
crmutils.str2file(
utillib.get_command_info(cmd)[1],
os.path.join(workdir, constants.CIB_TXT_F))
def analyze():
workdir = constants.WORKDIR
out_string = ""
tmp_string = ""
flist = [
constants.MEMBERSHIP_F, constants.CRM_MON_F, constants.B_CONF,
constants.SYSINFO_F, constants.CIB_F
]
for f in flist:
out_string += "Diff %s... " % f
if not glob.glob("%s/*/%s" % (workdir, f)):
out_string += "no %s/*/%s :/\n" % (workdir, f)
continue
code, tmp_string = analyze_one(workdir, f)
if tmp_string:
out_string += "\n" + tmp_string + "\n\n"
else:
out_string += "OK\n"
if code == 0:
if f != constants.CIB_F:
consolidate(workdir, f)
out_string += "\n"
out_string += check_crmvfy(workdir)
out_string += check_backtraces(workdir)
out_string += check_permissions(workdir)
out_string += check_logs(workdir)
analyze_f = os.path.join(workdir, constants.ANALYSIS_F)
crmutils.str2file(out_string, analyze_f)
def sys_info():
"""
some basic system info and stats
"""
out_string = "#####Cluster info:\n"
out_string += cluster_info()
out_string += crmsh_info()
out_string += ra_build_info()
out_string += crm_info()
out_string += booth_info()
out_string += "\n"
out_string += "#####Cluster related packages:\n"
out_string += pkg_versions(constants.PACKAGES)
if not constants.SKIP_LVL:
out_string += verify_packages(constants.PACKAGES)
out_string += "\n"
out_string += "#####System info:\n"
out_string += "Platform: %s\n" % os.uname()[0]
out_string += "Kernel release: %s\n" % os.uname()[2]
out_string += "Architecture: %s\n" % os.uname()[-1]
if os.uname()[0] == "Linux":
out_string += "Distribution: %s\n" % distro()
sys_info_f = os.path.join(constants.WORKDIR, constants.SYSINFO_F)
crmutils.str2file(out_string, sys_info_f)
def dump_logset(logf, from_time, to_time, outf):
"""
find log/set of logs which are interesting for us
"""
if os.stat(logf).st_size == 0:
return
logf_set = arch_logs(logf, from_time, to_time)
if not logf_set:
return
num_logs = len(logf_set)
oldest = logf_set[-1]
newest = logf_set[0]
mid_logfiles = logf_set[1:-1]
out_string = ""
# the first logfile: from $from_time to $to_time (or end)
# logfiles in the middle: all
# the last logfile: from beginning to $to_time (or end)
if num_logs == 1:
out_string += print_logseg(newest, from_time, to_time)
else:
out_string += print_logseg(oldest, from_time, 0)
for f in mid_logfiles:
out_string += print_log(f)
log_debug("including complete %s logfile" % f)
out_string += print_logseg(newest, 0, to_time)
crmutils.str2file(out_string, outf)
def collect_time_status():
out_string = "Time: "
out_string += datetime.datetime.now().strftime('%c') + '\n'
out_string += "ntpdc: "
out_string += utillib.get_command_info("ntpdc -pn")[1] + '\n'
time_f = os.path.join(constants.WORKDIR, constants.TIME_F)
crmutils.str2file(out_string, time_f)
def corosync_blackbox():
fdata_list = []
for f in find_files("/var/lib/corosync", constants.FROM_TIME, constants.TO_TIME):
if re.search("fdata", f):
fdata_list.append(f)
if fdata_list:
blackbox_f = os.path.join(constants.WORKDIR, constants.COROSYNC_RECORDER_F)
crmutils.str2file(get_command_info("corosync-blackbox")[1], blackbox_f)
def sys_stats():
out_string = ""
cmd_list = ["hostname", "uptime", "ps axf", "ps auxw", "top -b -n 1",\
"ip addr", "netstat -i", "arp -an", "lsscsi", "lspci",\
"mount", "cat /proc/cpuinfo", "df"]
for cmd in cmd_list:
out_string += "##### run \"%s\" on %s\n" % (cmd, constants.WE)
if cmd != "df":
out_string += get_command_info(cmd)[1] + '\n'
else:
out_string += get_command_info_timeout(cmd) + '\n'
sys_stats_f = os.path.join(constants.WORKDIR, constants.SYSSTATS_F)
crmutils.str2file(out_string, sys_stats_f)
def get_config():
workdir = constants.WORKDIR
if os.path.isfile(constants.CONF):
shutil.copy2(constants.CONF, workdir)
if crmutils.is_process("crmd"):
dump_state(workdir)
open(os.path.join(workdir, "RUNNING"), 'w')
else:
shutil.copy2(os.path.join(constants.CIB_DIR, constants.CIB_F), workdir)
open(os.path.join(workdir, "STOPPED"), 'w')
if os.path.isfile(os.path.join(workdir, constants.CIB_F)):
cmd = "crm_verify -V -x %s" % os.path.join(workdir, constants.CIB_F)
crmutils.str2file(
get_command_info(cmd)[1],
os.path.join(workdir, constants.CRM_VERIFY_F))
def mktemplate(argv):
workdir = constants.WORKDIR
out_string = constants.EMAIL_TMPLATE.format("%s"%date(), ' '.join(argv[1:]))
sysinfo_f = os.path.join(workdir, constants.SYSINFO_F)
if os.path.isfile(sysinfo_f):
out_string += "Common saystem info found:\n"
with open(sysinfo_f, 'r') as f:
out_string += f.read()
else:
for n in constants.NODES.split():
sysinfo_node_f = os.path.join(workdir, n, constants.SYSINFO_F)
if os.path.isfile(sysinfo_node_f):
out_string += "System info %s:\n" % n
out_string += sed_inplace(sysinfo_node_f, r'^', ' ')
out_string += "\n"
crmutils.str2file(out_string, os.path.join(workdir, constants.DESCRIPTION_F))
def collect_dlm_info():
"""
get dlm info
"""
if utillib.which("dlm_tool"):
out_string = "##### NOTICE - Lockspace overview:\n"
out_string += utillib.get_command_info("dlm_tool ls")[1] + '\n'
for item in utillib.grep("^name", incmd="dlm_tool ls"):
lock_name = item.split()[1]
out_string += "## NOTICE - Lockspace {}\n".format(lock_name)
out_string += utillib.get_command_info(
"dlm_tool lockdebug {}".format(lock_name))[1] + '\n'
out_string += "##### NOTICE - Lockspace history:\n"
out_string += utillib.get_command_info("dlm_tool dump")[1] + '\n'
dlm_f = os.path.join(constants.WORKDIR, constants.DLM_DUMP_F)
crmutils.str2file(out_string, dlm_f)
def collect_journal(from_t, to_t, outf):
if not which("journalctl"):
log_warning("Command journalctl not found")
return
if crmutils.is_int(from_t) and from_t == 0:
from_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
elif crmutils.is_int(from_t):
from_time = ts_to_dt(from_t).strftime("%Y-%m-%d %H:%M")
if crmutils.is_int(to_t) and to_t == 0:
to_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
elif crmutils.is_int(to_t):
to_time = ts_to_dt(to_t).strftime("%Y-%m-%d %H:%M")
if os.path.isfile(outf):
log_warning("%s already exists" % outf)
log_debug("journalctl from: '%d' until: '%d' from_time: '%s' to_time: '%s' > %s" % \
(from_t, to_t, from_time, to_time, outf))
cmd = 'journalctl -o short-iso --since "%s" --until "%s" --no-pager | tail -n +2' % \
(from_time, to_time)
crmutils.str2file(get_command_info(cmd)[1], outf)
def dump_state(workdir):
res = grep("^Last upd", incmd="crm_mon -1", flag="v")
crmutils.str2file('\n'.join(res), os.path.join(workdir, constants.CRM_MON_F))
cmd = "cibadmin -Ql"
crmutils.str2file(get_command_info(cmd)[1], os.path.join(workdir, constants.CIB_F))
cmd = "crm_node -p"
crmutils.str2file(get_command_info(cmd)[1], os.path.join(workdir, constants.MEMBERSHIP_F))
def events(destdir):
events_f = os.path.join(destdir, "events.txt")
out_string = ""
pattern = '|'.join(constants.EVENT_PATTERNS.split()[1::2])
halog_f = os.path.join(destdir, constants.HALOG_F)
if os.path.isfile(halog_f):
out_string = '\n'.join(grep(pattern, infile=halog_f))
crmutils.str2file(out_string, events_f)
for n in constants.NODES.split():
if os.path.isdir(os.path.join(destdir, n)):
events_node_f = os.path.join(destdir, n, "events.txt")
out_string = '\n'.join(grep(" %s "%n, infile=events_f))
crmutils.str2file(out_string, events_node_f)
else:
for n in constants.NODES.split():
halog_f = os.path.join(destdir, n, constants.HALOG_F)
if not os.path.isfile(halog_f):
continue
out_string = '\n'.join(grep(pattern, infile=halog_f))
crmutils.str2file(out_string, os.path.join(destdir, n, "events.text"))
def log_size(logf, outf):
l_size = os.stat(logf).st_size + 1
out_string = "%s %d" % (logf, l_size)
crmutils.str2file(out_string, outf)
