def collect_perms_state(): out_string = "" for check_dir in [ constants.PCMK_LIB, constants.PE_STATE_DIR, constants.CIB_DIR ]: flag = 0 out_string += "##### Check perms for %s: " % check_dir stat_info = os.stat(check_dir) if not stat.S_ISDIR(stat_info.st_mode): flag = 1 out_string += "\n%s wrong type or doesn't exist\n" % check_dir continue if stat_info.st_uid != pwd.getpwnam('hacluster')[2] or\ stat_info.st_gid != pwd.getpwnam('hacluster')[3] or\ "%04o" % (stat_info.st_mode & 0o7777) != "0750": flag = 1 out_string += "\nwrong permissions or ownership for %s: " % check_dir out_string += utillib.get_command_info( "ls -ld %s" % check_dir)[1] + '\n' if flag == 0: out_string += "OK\n" perms_f = os.path.join(constants.WORKDIR, constants.PERMISSIONS_F) crmutils.str2file(out_string, perms_f)
def collect_crm_config(): workdir = constants.WORKDIR if os.path.isfile(os.path.join(workdir, constants.CIB_F)): cmd = r"CIB_file=%s/%s crm configure show" % (workdir, constants.CIB_F) crmutils.str2file( utillib.get_command_info(cmd)[1], os.path.join(workdir, constants.CIB_TXT_F))
def analyze(): workdir = constants.WORKDIR out_string = "" tmp_string = "" flist = [ constants.MEMBERSHIP_F, constants.CRM_MON_F, constants.B_CONF, constants.SYSINFO_F, constants.CIB_F ] for f in flist: out_string += "Diff %s... " % f if not glob.glob("%s/*/%s" % (workdir, f)): out_string += "no %s/*/%s :/\n" % (workdir, f) continue code, tmp_string = analyze_one(workdir, f) if tmp_string: out_string += "\n" + tmp_string + "\n\n" else: out_string += "OK\n" if code == 0: if f != constants.CIB_F: consolidate(workdir, f) out_string += "\n" out_string += check_crmvfy(workdir) out_string += check_backtraces(workdir) out_string += check_permissions(workdir) out_string += check_logs(workdir) analyze_f = os.path.join(workdir, constants.ANALYSIS_F) crmutils.str2file(out_string, analyze_f)
def sys_info(): """ some basic system info and stats """ out_string = "#####Cluster info:\n" out_string += cluster_info() out_string += crmsh_info() out_string += ra_build_info() out_string += crm_info() out_string += booth_info() out_string += "\n" out_string += "#####Cluster related packages:\n" out_string += pkg_versions(constants.PACKAGES) if not constants.SKIP_LVL: out_string += verify_packages(constants.PACKAGES) out_string += "\n" out_string += "#####System info:\n" out_string += "Platform: %s\n" % os.uname()[0] out_string += "Kernel release: %s\n" % os.uname()[2] out_string += "Architecture: %s\n" % os.uname()[-1] if os.uname()[0] == "Linux": out_string += "Distribution: %s\n" % distro() sys_info_f = os.path.join(constants.WORKDIR, constants.SYSINFO_F) crmutils.str2file(out_string, sys_info_f)
def dump_logset(logf, from_time, to_time, outf): """ find log/set of logs which are interesting for us """ if os.stat(logf).st_size == 0: return logf_set = arch_logs(logf, from_time, to_time) if not logf_set: return num_logs = len(logf_set) oldest = logf_set[-1] newest = logf_set[0] mid_logfiles = logf_set[1:-1] out_string = "" # the first logfile: from $from_time to $to_time (or end) # logfiles in the middle: all # the last logfile: from beginning to $to_time (or end) if num_logs == 1: out_string += print_logseg(newest, from_time, to_time) else: out_string += print_logseg(oldest, from_time, 0) for f in mid_logfiles: out_string += print_log(f) log_debug("including complete %s logfile" % f) out_string += print_logseg(newest, 0, to_time) crmutils.str2file(out_string, outf)
def collect_time_status(): out_string = "Time: " out_string += datetime.datetime.now().strftime('%c') + '\n' out_string += "ntpdc: " out_string += utillib.get_command_info("ntpdc -pn")[1] + '\n' time_f = os.path.join(constants.WORKDIR, constants.TIME_F) crmutils.str2file(out_string, time_f)
def corosync_blackbox(): fdata_list = [] for f in find_files("/var/lib/corosync", constants.FROM_TIME, constants.TO_TIME): if re.search("fdata", f): fdata_list.append(f) if fdata_list: blackbox_f = os.path.join(constants.WORKDIR, constants.COROSYNC_RECORDER_F) crmutils.str2file(get_command_info("corosync-blackbox")[1], blackbox_f)
def sys_stats(): out_string = "" cmd_list = ["hostname", "uptime", "ps axf", "ps auxw", "top -b -n 1",\ "ip addr", "netstat -i", "arp -an", "lsscsi", "lspci",\ "mount", "cat /proc/cpuinfo", "df"] for cmd in cmd_list: out_string += "##### run \"%s\" on %s\n" % (cmd, constants.WE) if cmd != "df": out_string += get_command_info(cmd)[1] + '\n' else: out_string += get_command_info_timeout(cmd) + '\n' sys_stats_f = os.path.join(constants.WORKDIR, constants.SYSSTATS_F) crmutils.str2file(out_string, sys_stats_f)
def get_config(): workdir = constants.WORKDIR if os.path.isfile(constants.CONF): shutil.copy2(constants.CONF, workdir) if crmutils.is_process("crmd"): dump_state(workdir) open(os.path.join(workdir, "RUNNING"), 'w') else: shutil.copy2(os.path.join(constants.CIB_DIR, constants.CIB_F), workdir) open(os.path.join(workdir, "STOPPED"), 'w') if os.path.isfile(os.path.join(workdir, constants.CIB_F)): cmd = "crm_verify -V -x %s" % os.path.join(workdir, constants.CIB_F) crmutils.str2file( get_command_info(cmd)[1], os.path.join(workdir, constants.CRM_VERIFY_F))
def mktemplate(argv): workdir = constants.WORKDIR out_string = constants.EMAIL_TMPLATE.format("%s"%date(), ' '.join(argv[1:])) sysinfo_f = os.path.join(workdir, constants.SYSINFO_F) if os.path.isfile(sysinfo_f): out_string += "Common saystem info found:\n" with open(sysinfo_f, 'r') as f: out_string += f.read() else: for n in constants.NODES.split(): sysinfo_node_f = os.path.join(workdir, n, constants.SYSINFO_F) if os.path.isfile(sysinfo_node_f): out_string += "System info %s:\n" % n out_string += sed_inplace(sysinfo_node_f, r'^', ' ') out_string += "\n" crmutils.str2file(out_string, os.path.join(workdir, constants.DESCRIPTION_F))
def collect_dlm_info(): """ get dlm info """ if utillib.which("dlm_tool"): out_string = "##### NOTICE - Lockspace overview:\n" out_string += utillib.get_command_info("dlm_tool ls")[1] + '\n' for item in utillib.grep("^name", incmd="dlm_tool ls"): lock_name = item.split()[1] out_string += "## NOTICE - Lockspace {}\n".format(lock_name) out_string += utillib.get_command_info( "dlm_tool lockdebug {}".format(lock_name))[1] + '\n' out_string += "##### NOTICE - Lockspace history:\n" out_string += utillib.get_command_info("dlm_tool dump")[1] + '\n' dlm_f = os.path.join(constants.WORKDIR, constants.DLM_DUMP_F) crmutils.str2file(out_string, dlm_f)
def collect_journal(from_t, to_t, outf): if not which("journalctl"): log_warning("Command journalctl not found") return if crmutils.is_int(from_t) and from_t == 0: from_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") elif crmutils.is_int(from_t): from_time = ts_to_dt(from_t).strftime("%Y-%m-%d %H:%M") if crmutils.is_int(to_t) and to_t == 0: to_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") elif crmutils.is_int(to_t): to_time = ts_to_dt(to_t).strftime("%Y-%m-%d %H:%M") if os.path.isfile(outf): log_warning("%s already exists" % outf) log_debug("journalctl from: '%d' until: '%d' from_time: '%s' to_time: '%s' > %s" % \ (from_t, to_t, from_time, to_time, outf)) cmd = 'journalctl -o short-iso --since "%s" --until "%s" --no-pager | tail -n +2' % \ (from_time, to_time) crmutils.str2file(get_command_info(cmd)[1], outf)
def dump_state(workdir): res = grep("^Last upd", incmd="crm_mon -1", flag="v") crmutils.str2file('\n'.join(res), os.path.join(workdir, constants.CRM_MON_F)) cmd = "cibadmin -Ql" crmutils.str2file(get_command_info(cmd)[1], os.path.join(workdir, constants.CIB_F)) cmd = "crm_node -p" crmutils.str2file(get_command_info(cmd)[1], os.path.join(workdir, constants.MEMBERSHIP_F))
def events(destdir): events_f = os.path.join(destdir, "events.txt") out_string = "" pattern = '|'.join(constants.EVENT_PATTERNS.split()[1::2]) halog_f = os.path.join(destdir, constants.HALOG_F) if os.path.isfile(halog_f): out_string = '\n'.join(grep(pattern, infile=halog_f)) crmutils.str2file(out_string, events_f) for n in constants.NODES.split(): if os.path.isdir(os.path.join(destdir, n)): events_node_f = os.path.join(destdir, n, "events.txt") out_string = '\n'.join(grep(" %s "%n, infile=events_f)) crmutils.str2file(out_string, events_node_f) else: for n in constants.NODES.split(): halog_f = os.path.join(destdir, n, constants.HALOG_F) if not os.path.isfile(halog_f): continue out_string = '\n'.join(grep(pattern, infile=halog_f)) crmutils.str2file(out_string, os.path.join(destdir, n, "events.text"))
def log_size(logf, outf): l_size = os.stat(logf).st_size + 1 out_string = "%s %d" % (logf, l_size) crmutils.str2file(out_string, outf)