def _lic_show(opts, act_conf): elo = sge_license_tools.ExternalLicenses(opts.base, opts.site, log_com=_log) _xml = sge_license_tools.LicenseCheck( log_com=_log, lmutil_path=act_conf["LMUTIL_PATH"], license_file=act_conf["LICENSE_FILE"], verbose=opts.verbose, ).check() elo.read() elo.feed_xml_result(_xml) out_list = logging_tools.NewFormList() for _t_type in ["simple", "complex"]: for _name in sorted(elo.licenses.keys()): _lic = elo.licenses[_name] if _lic.license_type == _t_type: out_list.append(_lic.get_info_line()) print(str(out_list))
def _log_limits(self): # read limits r_dict = {} try: import resource except ImportError: self.log("cannot import resource", logging_tools.LOG_LEVEL_CRITICAL) else: available_resources = [ key for key in dir(resource) if key.startswith("RLIMIT") ] for av_r in available_resources: try: r_dict[av_r] = resource.getrlimit(getattr(resource, av_r)) except ValueError: r_dict[av_r] = "invalid resource" except: r_dict[av_r] = None if r_dict: res_keys = sorted(r_dict.keys()) self.log("{} defined".format( logging_tools.get_plural("limit", len(res_keys)))) res_list = logging_tools.NewFormList() for key in res_keys: val = r_dict[key] if isinstance(val, str): info_str = val elif isinstance(val, tuple): info_str = "{:8d} (hard), {:8d} (soft)".format(*val) else: info_str = "None (error?)" res_list.append([ logging_tools.form_entry(key, header="key"), logging_tools.form_entry(info_str, header="value") ]) for line in str(res_list).split("\n"): self.log(line) else: self.log("no limits found, strange ...", logging_tools.LOG_LEVEL_WARN)
def do_list(cur_opts, log_com): users = get_users(cur_opts, log_com) out_list = logging_tools.NewFormList() for _user in users: out_list.append( [ logging_tools.form_entry(_user.login, header="login"), logging_tools.form_entry(_user.uid, header="uid"), logging_tools.form_entry(_user.active, header="active"), logging_tools.form_entry(_user.group.groupname, header="group"), logging_tools.form_entry(_user.group.gid, header="gid"), logging_tools.form_entry(_user.group.active, header="gactive"), logging_tools.form_entry(_user.first_name, header="first name"), logging_tools.form_entry(_user.last_name, header="last name"), logging_tools.form_entry(_user.email, header="email"), logging_tools.form_entry(_user.login_count, header="logincount"), logging_tools.form_entry(_user.login_fail_count, header="failedcount"), logging_tools.form_entry(_user.user_variable_set.all().count(), header="#vars"), logging_tools.form_entry(_user.comment, header="comment"), ] ) print(str(out_list))
def get_config_info(self): gk = sorted(self.keys()) if gk: f_obj = logging_tools.NewFormList() for key in gk: if self.get_type(key) in ["a", "d"]: pv = self.pretty_print(key) f_obj.append([ logging_tools.form_entry(key), logging_tools.form_entry("list with {}:".format( logging_tools.get_plural("entry", len(pv)))), logging_tools.form_entry(self.get_type(key)), logging_tools.form_entry(self.get_source(key)), ]) for idx, entry in enumerate(pv): f_obj.append([ logging_tools.form_entry(""), logging_tools.form_entry(entry), logging_tools.form_entry(str(idx)), logging_tools.form_entry("---"), ]) else: f_obj.append([ logging_tools.form_entry(key, header="key"), logging_tools.form_entry(self.pretty_print(key), header="value"), logging_tools.form_entry(self.get_type(key), pre_str=", (", post_str=" from ", header="type"), logging_tools.form_entry(self.get_source(key), post_str=")", header="source"), ]) ret_str = str(f_obj).split("\n") else: ret_str = [] return ret_str
def _domain_enum_show_command(options): from initat.cluster.backbone.domain_enum import icswDomainEnum from initat.cluster.backbone.models import DomainTypeEnum print("") print("DomainEnums defined: {:d}".format(len(icswDomainEnum))) _list = logging_tools.NewFormList() _c_dict = { entry.enum_name: entry for entry in DomainTypeEnum.objects.all() } for entry in icswDomainEnum: if entry.name not in _c_dict: if options.sync: new_entry = DomainTypeEnum.create_db_entry(entry) _c_dict[new_entry.enum_name] = new_entry else: _db_str = "no" if entry.name in _c_dict: # if options.sync: # _c_dict[entry.name].update_values(entry) _db_str = "yes ({:d})".format(_c_dict[entry.name].pk) if entry.value.default_enum: _default_info = entry.value.default_enum.name else: _default_info = "---" if entry.value.domain_enum: _domain_info = entry.value.domain_enum.name else: _domain_info = "---" _list.append([ logging_tools.form_entry(entry.name, header="EnumName"), logging_tools.form_entry(entry.value.name, header="Name"), logging_tools.form_entry(entry.value.info, header="Info"), logging_tools.form_entry_center(_db_str, header="DB info"), logging_tools.form_entry(_default_info, header="Default Enum"), logging_tools.form_entry(_domain_info, header="Domain Enum"), ]) print(str(_list))
def interpret(self, srv_com, cur_ns): if cur_ns.arguments: re_list = [re.compile(_arg) for _arg in cur_ns.arguments] else: re_list = [] cur_vector = srv_com["data:machine_vector"] if cur_ns.raw: return limits.mon_STATE_OK, etree.tostring(cur_vector, encoding="unicode") else: vector_keys = sorted(srv_com.xpath(".//ns:mve/@name", start_el=cur_vector, smart_strings=False)) used_keys = [key for key in vector_keys if any([cur_re.search(key) for cur_re in re_list]) or not re_list] ret_array = [ "MachineVector id {}, {}, {} shown{}".format( cur_vector.attrib["version"], logging_tools.get_plural("key", len(vector_keys)), logging_tools.get_plural("key", len(used_keys)), ":" if used_keys else "", ) ] if used_keys: out_list = logging_tools.NewFormList() max_num_keys = 0 _list = [] a = 2 for mv_num, mv_key in enumerate(vector_keys): if mv_key in used_keys: cur_xml = srv_com.xpath("//ns:mve[@name='{}']".format(mv_key), start_el=cur_vector, smart_strings=False)[0] _mv = hm_classes.MachineVectorEntry( cur_xml.attrib.pop("name"), **cur_xml.attrib ) _list.append((mv_num, _mv)) max_num_keys = max(max_num_keys, _mv.num_keys) for mv_num, entry in _list: out_list.append(entry.get_form_entry(mv_num, max_num_keys)) ret_array.extend(str(out_list).split("\n")) return limits.mon_STATE_OK, "\n".join(ret_array)
def interpret(self, srv_com, cur_ns): _fe = logging_tools.form_entry def proc_line(_ps, **kwargs): nest = kwargs.get("nest", 0) if _psutil: _affinity = _ps["cpu_affinity"] if len(_affinity) == num_cores: _affinity = "-" else: _affinity = ",".join( ["{:d}".format(_core) for _core in _affinity]) pass else: _affinity = _ps.get("affinity", "-") return [ _fe("{}{:d}".format(" " * nest, _ps["pid"]), header="pid"), _fe(_ps["ppid"], header="ppid"), _fe(_ps["uids"][0] if _psutil else proc_stuff["uid"], header="uid"), _fe(_ps["gids"][0] if _psutil else proc_stuff["gid"], header="gid"), _fe(_ps["state"], header="state"), _fe(_ps.get("last_cpu", -1), header="cpu"), _fe(_affinity, header="aff"), _fe(_ps["out_name"], header="process"), ] def draw_tree(m_pid, nest=0): proc_stuff = result[m_pid] r_list = [proc_line(proc_stuff, nest=nest)] # _fe("%s%s" % (" " * nest, m_pid), header="pid"), for dt_entry in [ draw_tree(y, nest + 2) for y in result[m_pid]["childs"] ]: r_list.extend([z for z in dt_entry]) return r_list tree_view = cur_ns.tree comline_view = cur_ns.comline if cur_ns.filter: name_re = re.compile("^.*%s.*$" % ("|".join(cur_ns.filter)), re.IGNORECASE) tree_view = False else: name_re = re.compile(".*") result = srv_com["process_tree"] _psutil = "psutil" in srv_com if _psutil: num_cores = srv_com["*num_cores"] # unpack and cast pid to integer result = { int(key): value for key, value in server_command.decompress(result.text, json=True).items() } for _val in result.values(): _val["state"] = process_tools.PROC_STATUSES_REV[_val["status"]] # print etree.tostring(srv_com.tree, pretty_print=True) ret_state = limits.mon_STATE_CRITICAL pids = sorted([ key for key, value in result.items() if name_re.match(value["name"]) ]) for act_pid in pids: proc_stuff = result[act_pid] proc_name = proc_stuff["name"] if proc_stuff["exe"] else "[%s]" % ( proc_stuff["name"]) if comline_view: proc_name = " ".join(proc_stuff.get("cmdline")) or proc_name proc_stuff["out_name"] = proc_name ret_a = [ "found {} matching {}".format( logging_tools.get_plural("process", len(pids)), name_re.pattern) ] form_list = logging_tools.NewFormList() if tree_view: for act_pid in pids: result[act_pid]["childs"] = [ pid for pid in pids if result[pid]["ppid"] == int(act_pid) ] for init_pid in [pid for pid in pids if not result[pid]["ppid"]]: form_list.extend( [add_line for add_line in draw_tree(init_pid)]) else: form_list.extend([proc_line(result[_pid]) for _pid in pids]) if form_list: ret_a.extend(str(form_list).split("\n")) return ret_state, "\n".join(ret_a)
def do_info(cur_opts, log_com): if not cur_opts.username: print("No user name given") return 1 _user = _get_user(cur_opts.username) _ret_state = 0 if _user is None: _ret_state = 1 else: from initat.cluster.backbone.models import user_quota_setting, user_variable from django.db.models import Q print("") print( "User with loginname '{}' (user {}), uid={:d}, group={} (gid={:d})".format( _user.login, str(_user), _user.uid, str(_user.group), _user.group.gid, ) ) num_qs = _user.user_quota_setting_set.all().count() if num_qs and cur_opts.system_wide_quota: print("") print( "{} found:".format( logging_tools.get_plural("system-wide quota setting", num_qs) ) ) for _qs in _user.user_quota_setting_set.all(): _bd = _qs.quota_capable_blockdevice print( " device {} ({} on {}): {}".format( str(_bd.device.full_name), _bd.block_device_path, _bd.mount_path, get_quota_str(_qs), ) ) try: _cmd = "quota --show-mntpoint -wp -u {}".format( _user.login, ) _res = subprocess.check_output( _cmd.split(), stderr=subprocess.STDOUT, ).decode("utf-8") except subprocess.CalledProcessError as sb_exc: _res = sb_exc.output.decode("utf-8") # print("error calling '{}': {}".format(_cmd, process_tools.get_except_info())) _ret_state = 1 except OSError as sb_exc: # quota command not founda _res = "denied: {}".format(sb_exc) # print("error calling '{}': {}".format(_cmd, process_tools.get_except_info())) _ret_state = 1 else: _ret_state = 0 if _res.lower().count("denied"): print(" error getting local quotas for {}: {}".format(_user.login, _res)) else: # print _res _lines = [_line.strip().split() for _line in _res.split("\n") if _line.strip()] _lines = [_line for _line in _lines if len(_line) == 10] if _lines: print("", "local quota:", sep="\n") _line = _lines[-1] _bytes_violate = _line[2].count("*") > 0 _local = user_quota_setting( bytes_used=int(_line[2].replace("*", "")) * 1024, bytes_soft=int(_line[3]) * 1024, bytes_hard=int(_line[4]) * 1024, bytes_gracetime=int(_line[5]), ) print( " local mountpoint: {}".format( get_quota_str(_local), ) ) if cur_opts.delete_var: print("") try: _cv = user_variable.objects.get(Q(user=_user) & Q(idx=cur_opts.delete_var)) except user_variable.DoesNotExist: print("Variable to delete does not exist") else: print("Deleting '{}'".format(str(_cv))) _cv.delete() if cur_opts.show_vars: out_list = logging_tools.NewFormList() for _var in _user.user_variable_set.all().order_by("name"): out_list.append( [ logging_tools.form_entry(_var.idx, header="idx"), logging_tools.form_entry(_var.name, header="name"), logging_tools.form_entry(_var.var_type, header="type"), logging_tools.form_entry_right(_var.value if _var.var_type != "j" else "{:d} Bytes".format(len(_var.json_value)), header="value"), logging_tools.form_entry_center("yes" if _var.editable else "no", header="editable"), logging_tools.form_entry_center("yes" if _var.hidden else "no", header="hidden"), logging_tools.form_entry(_var.date.strftime("%H:%m:%S %a, %d. %b %Y"), header="created"), logging_tools.form_entry(_var.description, header="description"), ] ) print(str(out_list)) return _ret_state
def interpret(self, srv_com, cur_ns): r_dict = server_command.decompress(srv_com["pkg_list"].text, pickle=True) root_dir = srv_com["root_dir"].text in_format = srv_com["format"].text out_f = logging_tools.NewFormList() keys = sorted(r_dict.keys()) header_line = "{} found, system is {} (root is {})".format( logging_tools.get_plural("package", len(keys)), in_format, root_dir, ) if keys: if in_format == "rpm": for key in keys: for value in r_dict[key]: if isinstance(value, tuple): if len(value) == 4: ver, rel, arch, summary = value size = 0 else: ver, rel, arch, size, summary = value else: ver, rel, arch, size, summary = (value["version"], value["release"], value["arch"], value["size"], value["summary"]) out_f.append([ logging_tools.form_entry(key, header="name"), logging_tools.form_entry_right(ver, header="version"), logging_tools.form_entry(rel, header="release"), logging_tools.form_entry(arch, header="arch"), logging_tools.form_entry_right(size, header="size"), logging_tools.form_entry(summary, header="summary"), ]) elif in_format == "debian": for key in keys: for value in r_dict[key]: d_flag, s_flag, e_flag = value["flags"] ver, rel = (value["version"], value["release"]) summary = value["summary"] out_f.append([ logging_tools.form_entry(key, header="name"), logging_tools.form_entry_right(d_flag, header="d_flag"), logging_tools.form_entry_right(s_flag, header="s_flag"), logging_tools.form_entry_right(e_flag, header="e_flag"), logging_tools.form_entry_right(ver, header="version"), logging_tools.form_entry(rel, header="release"), logging_tools.form_entry(summary, header="summary"), ]) out_f.add_line( (key, d_flag, s_flag, e_flag, ver, rel, summary)) return limits.mon_STATE_OK, "{}\n{}".format( header_line, str(out_f)) else: return limits.mon_STATE_CRITICAL, "{}, nothing found".format( header_line)
def _service_enum_show_command(options): from initat.cluster.backbone.server_enums import icswServiceEnum from initat.cluster.backbone.models import ConfigServiceEnum, config from initat.cluster.backbone import factories from django.core.exceptions import ValidationError _c_dict = { entry.enum_name: entry for entry in ConfigServiceEnum.objects.all() } print("") print("ServiceEnums defined: {:d}".format(len(icswServiceEnum))) _list = logging_tools.NewFormList() for entry in icswServiceEnum: if entry.name not in _c_dict: if options.sync and (entry.value.server_service or entry.value.relayer_service ) and entry.value.sync_config: new_entry = ConfigServiceEnum.create_db_entry(entry) _c_dict[new_entry.enum_name] = new_entry else: _db_str = "no" if entry.name in _c_dict: if options.sync: _c_dict[entry.name].update_values(entry) _db_str = "yes ({:d})".format(_c_dict[entry.name].pk) if entry.value.server_service: _egg_action = ", ".join( [str(_action) for _action in entry.value.egg_actions]) or "none" else: _egg_action = "---" _list.append([ logging_tools.form_entry(entry.name, header="EnumName"), logging_tools.form_entry(entry.value.name, header="Name"), logging_tools.form_entry_center( "yes" if entry.value.root_service else "no", header="Root Service"), logging_tools.form_entry_center( "yes" if entry.value.server_service else "no", header="Server"), logging_tools.form_entry_center( "yes" if entry.value.relayer_service else "no", header="Relayer"), logging_tools.form_entry(entry.value.info, header="Info"), logging_tools.form_entry_center(_db_str, header="DB info"), logging_tools.form_entry(_egg_action, header="Egg actions"), ]) print(str(_list)) if options.sync: _change_list = [] # compat dict comp_dict = { "rrd_grapher": icswServiceEnum.grapher_server.name, "rrd_server": icswServiceEnum.collectd_server.name, "rrd_collector": icswServiceEnum.collectd_server.name, "server": icswServiceEnum.cluster_server.name, "ldap_server": icswServiceEnum.ldap_server.name, } for c_con in config.objects.all(): if not c_con.config_service_enum_id: _check_names = [c_con.name] if c_con.name in comp_dict: _check_names.append(comp_dict[c_con.name]) for _check_name in _check_names: if _check_name in _c_dict: c_con.config_service_enum = _c_dict[_check_name] try: c_con.save(update_fields=["config_service_enum"]) except ValidationError: print("cannot save {}: {}".format( str(c_con), process_tools.get_except_info())) else: _change_list.append(c_con) break _create_list = [] for db_enum in _c_dict.values(): if not db_enum.config_set.all().count(): _create_list.append( factories.Config( name=db_enum.name, description=db_enum.info, config_service_enum=db_enum, server_config=True, )) if len(_change_list): print("") print("{} moved to ConfigServiceEnum:".format( logging_tools.get_plural("Config", len(_change_list)))) for entry in _change_list: print(" {} ({})".format(entry.name, str(entry.config_service_enum))) if len(_create_list): print("") print("{} created:".format( logging_tools.get_plural("Config", len(_create_list)))) for entry in _create_list: print(" {} ({})".format(entry.name, str(entry.config_service_enum)))
def main(options): options.overview = True if (not options.stat and not options.index and not options.num) else False options.index = [int(cur_idx) for cur_idx in options.index] err_file_name = os.path.join(LOG_ROOT, "logging-server", "err_py") if not os.path.isfile(err_file_name): print("{} does not exist".format(err_file_name)) sys.exit(1) if options.clear: new_file_name = "{}_{}.tar".format( err_file_name, time.strftime("%Y-%m-%d_%H:%M:%S", time.localtime()) ) if process_tools.find_file("xz"): _pf = ".xz" _compr = "J" c_stat, out = subprocess.getstatusoutput( "tar cpJf {}{} {}".format( new_file_name, _pf, err_file_name ) ) elif process_tools.find_file("bzip2"): _pf = ".bz2" _compr = "j" c_stat, out = subprocess.getstatusoutput( "tar cpjf {}{} {}".format( new_file_name, _pf, err_file_name ) ) else: _pf = "" _compr = "" print( "taring {} to {}{} ...".format( err_file_name, new_file_name, _pf ) ) c_stat, out = subprocess.getstatusoutput( "tar cp{}f {}{} {}".format( _compr, new_file_name, _pf, err_file_name ) ) if c_stat: print("*** error (%d): %s" % (c_stat, out)) else: os.unlink(err_file_name) sys.exit(c_stat) try: err_lines = [ line.strip() for line in open( err_file_name, "r" ).read().split("\n") if line.count("[ES ") ] except IOError: print( "Cannot read '{}': {}".format( err_file_name, process_tools.get_except_info() ) ) sys.exit(1) print( "Found error_file {} with {}".format( err_file_name, logging_tools.get_plural("line", len(err_lines)) ) ) errs_found, act_err = ([], None) act_idx, idx_dict, prev_dt = (0, {}, None) for line in err_lines: line_parts = line.split(":") _chunk = line_parts[-1].split()[-1] # date is always the first 4 parts line_date = ":".join(line_parts[0:3]).strip() info_part = line_parts[3].strip() err_line = ":".join(line_parts[4:]) # parse info_part try: if info_part.startswith("("): line_state = "" else: line_state = info_part.split()[0] info_part = info_part[len(line_state):].strip() info_parts = info_part.split() # skip error-thread name and "from pid" string info_parts.pop(0) info_parts.pop(0) info_parts.pop(0) except: print( "Error pre-parsing line '{}': {}".format( line, process_tools.get_except_info() ) ) else: try: _struct = json.loads(bz2.decompress(base64.b64decode(_chunk))) # get pid line_pid = _struct["pid"] # unknown or full source line_s_name = _struct["name"] line_uid = _struct["uid"] line_gid = _struct["gid"] line_uname = _struct["uname"] line_gname = _struct["gname"] err_line = _struct["line"] cur_dt = datetime.datetime.strptime(line_date, "%a %b %d %H:%M:%S %Y") if prev_dt: dt_change = abs(cur_dt - prev_dt).seconds > 5 else: dt_change = False prev_dt = cur_dt if not act_err or act_err.pid != line_pid or dt_change or line.count("<type"): act_idx += 1 act_err = ErrorRecord( line_pid, line_s_name, line_uid, line_uname, line_gid, line_gname, ) act_err.set_idx(act_idx) idx_dict[act_idx] = act_err errs_found.append(act_err) if err_line.strip() or not options.noempty: act_err.add_line(line_date, line_state, err_line) except: print( "Error parsing line '{}': {}".format( line, process_tools.get_except_info() ) ) print( "Found {}".format( logging_tools.get_plural("error record", len(errs_found)) ) ) if options.overview: if errs_found: out_list = logging_tools.NewFormList() for err in errs_found: out_list.append(err.get_form_parts()) print(str(out_list)) elif options.stat: uid_dict = {} for err in errs_found: uid_dict.setdefault(err.uid, []).append(err) all_uids = list(uid_dict.keys()) all_uids.sort() out_list = logging_tools.NewFormList() for uid in all_uids: uid_stuff = uid_dict[uid] diff_sources = [] for err in uid_stuff: if err.source_name not in diff_sources: diff_sources.append(err.source_name) diff_sources.sort() out_list.append( [ logging_tools.form_entry(uid, header="uid"), logging_tools.form_entry(uid_stuff[0].uname, header="uname"), logging_tools.form_entry(len(uid_stuff), header="# err"), logging_tools.form_entry(len(diff_sources), header="# sources"), logging_tools.form_entry(", ".join(diff_sources), header="sources"), ] ) print(str(out_list)) elif options.num: idx_l = list(idx_dict.keys()) idx_l.sort() idx_show = [] while options.num and idx_l: options.num -= 1 idx_show.append(idx_l.pop(-1)) idx_show.reverse() options.index = idx_show if options.index: for idx in options.index: if idx in idx_dict: act_err = idx_dict[idx] print(act_err.get_header()) print(act_err.show_lines()) else: print( "Index {:d} not in index_list {}".format( idx, logging_tools.compress_num_list(list(idx_dict.keys())) ) )
def _call(self, cur_inst): file_list = [] server_idxs = [self.server_idx] # get additional idx if host is virtual server sc_result = config_tools.icswServerCheck(service_type_enum=icswServiceEnum.cluster_server).get_result() if sc_result.effective_device is not None and sc_result.effective_device.idx != self.server_idx: server_idxs.append(sc_result.effective_device.idx) # recognize for which devices i am responsible dev_r = cluster_location.DeviceRecognition() server_idxs = list(set(server_idxs) | set(dev_r.device_dict.keys())) # get all peers to local machine and local netdevices my_idxs = netdevice.objects.exclude( Q(enabled=False) ).filter( Q(device__in=server_idxs) & Q(device__enabled=True) & Q(device__device_group__enabled=True) ).values_list("pk", flat=True) # ref_table route_obj = config_tools.RouterObject(cur_inst.log) all_paths = [] for s_ndev in my_idxs: all_paths.extend(list(networkx.shortest_path(route_obj.nx, s_ndev, weight="weight").values())) # pprint.pprint(all_paths) nd_lut = { cur_nd.pk: cur_nd for cur_nd in netdevice.objects.all().select_related( "device" ).prefetch_related( "net_ip_set", "net_ip_set__network", "net_ip_set__domain_tree_node" ) } # fetch key-information ssh_vars = device_variable.objects.filter(Q(name="ssh_host_rsa_key_pub")).select_related("device") rsa_key_dict = {} for _db_rec in ssh_vars: pass # read pre/post lines from /etc/hosts pre_host_lines, post_host_lines = ([], []) # parse pre/post host_lines try: host_lines = [line.strip() for line in codecs.open(ETC_HOSTS_FILENAME, "r", "utf-8").read().split("\n")] except: self.log( "error reading / parsing {}: {}".format( ETC_HOSTS_FILENAME, process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR) else: mode, any_modes_found = (0, False) for line in host_lines: if line.lower().startswith("### aeh-start-pre"): mode, any_modes_found = (1, True) elif line.lower().startswith("### aeh-start-post"): mode, any_modes_found = (2, True) elif line.lower().startswith("### aeh-end"): mode, any_modes_found = (0, True) else: if mode == 1: pre_host_lines.append(line) elif mode == 2: post_host_lines.append(line) if not any_modes_found: self.log( "no ### aeh-.* stuff found in {}, copying to {}.orig".format( ETC_HOSTS_FILENAME, ETC_HOSTS_FILENAME ) ) try: pass except: self.log( "error writing {}.orig: {}".format( ETC_HOSTS_FILENAME, process_tools.get_except_info() ) ) # mapping from device_name to all names for ssh_host_keys name_dict = {} # ip dictionary ip_dict = {} # min_target_dict min_target_dict = {} for cur_path in all_paths: min_value = route_obj.get_penalty(cur_path) target_nd = nd_lut[cur_path[-1]] min_target_dict[target_nd] = min(min_target_dict.get(target_nd, 999999999), min_value) tl_dtn = domain_tree_node.objects.get(Q(depth=0)) for cur_path in all_paths: target_nd = nd_lut[cur_path[-1]] min_value = min_target_dict[target_nd] for cur_ip in nd_lut[cur_path[-1]].net_ip_set.all(): # get names host_names = [] cur_dtn = cur_ip.domain_tree_node or tl_dtn if not (cur_ip.alias.strip() and cur_ip.alias_excl): host_names.append("{}{}".format(target_nd.device.name, cur_dtn.node_postfix)) host_names.extend(["{}".format(cur_entry) for cur_entry in cur_ip.alias.strip().split()]) if "localhost" in [x.split(".")[0] for x in host_names]: host_names = [host_name for host_name in host_names if host_name.split(".")[0] == "localhost"] if cur_dtn.full_name: if cur_dtn.create_short_names: # also create short_names out_names = ( " ".join( [ "{}.{} {}".format(host_name, cur_dtn.full_name, host_name) for host_name in host_names if not host_name.count(".") ] ) ).split() else: # only print the long names out_names = ["{}.{}".format(host_name, cur_dtn.full_name) for host_name in host_names if not host_name.count(".")] else: if cur_dtn.create_short_names: # also create short_names out_names = (" ".join(["{}".format(host_name) for host_name in host_names if not host_name.count(".")])).split() else: # only print the long names out_names = ["{}".format(host_name) for host_name in host_names if not host_name.count(".")] # add names with dot out_names.extend([host_name for host_name in host_names if host_name.count(".")]) # name_dict without localhost name_dict.setdefault( target_nd.device.name, [] ).extend( [ out_name for out_name in out_names if out_name not in name_dict[target_nd.device.name] and not out_name.startswith("localhost") ] ) ip_dict.setdefault(cur_ip.ip, []) if out_names not in [entry[1] for entry in ip_dict[cur_ip.ip]]: if cur_ip.ip != "0.0.0.0": ip_dict[cur_ip.ip].append((min_value, out_names)) # out_list loc_dict = {} for ip, h_list in ip_dict.items(): all_values = sorted([entry[0] for entry in h_list]) if all_values: min_value = all_values[0] out_names = [] for val in all_values: for _act_val, act_list in [(x_value, x_list) for x_value, x_list in h_list if x_value == val]: out_names.extend([value for value in act_list if value not in out_names]) # print min_value, ip, out_names loc_dict.setdefault(min_value, []).append([ipvx_tools.IPv4(ip)] + out_names) pen_list = sorted(loc_dict.keys()) out_file = [] for pen_value in pen_list: act_out_list = logging_tools.NewFormList() for entry in sorted(loc_dict[pen_value]): act_out_list.append( [ logging_tools.form_entry(entry[0]) ] + [ logging_tools.form_entry(cur_e) for cur_e in entry[1:] ] ) host_lines = str(act_out_list).split("\n") out_file.extend( [ "# penalty {:d}, {}".format( pen_value, logging_tools.get_plural("host entry", len(host_lines)) ), "" ] + host_lines + [""] ) if not os.path.isdir(GROUP_DIR): try: os.makedirs(GROUP_DIR) except: pass if os.path.isdir(GROUP_DIR): # remove old files for file_name in os.listdir(GROUP_DIR): try: os.unlink(os.path.join(GROUP_DIR, file_name)) except: pass # get all devices with netips all_devs = device.objects.filter( Q(enabled=True) & Q(device_group__enabled=True) & Q(netdevice__net_ip__ip__contains=".") ).values_list( "name", "device_group__name" ).order_by( "device_group__name", "name" ) dg_dict = {} for dev_name, dg_name in all_devs: dg_dict.setdefault(dg_name, []).append(dev_name) for file_name, content in dg_dict.items(): codecs.open( os.path.join(GROUP_DIR, file_name), "w", "utf-8" ).write("\n".join(sorted(set(content)) + [""])) file_list.append(ETC_HOSTS_FILENAME) codecs.open(ETC_HOSTS_FILENAME, "w+", "utf-8").write( "\n".join( [ "### AEH-START-PRE insert pre-host lines below" ] + pre_host_lines + [ "### AEH-END-PRE insert pre-host lines above", "" ] + out_file + [ "", "### AEH-START-POST insert post-host lines below" ] + post_host_lines + [ "### AEH-END-POST insert post-host lines above", "" ] ) ) # write known_hosts_file if os.path.isdir(os.path.dirname(SSH_KNOWN_HOSTS_FILENAME)): skh_f = open(SSH_KNOWN_HOSTS_FILENAME, "w") for ssh_key_node in sorted(rsa_key_dict.keys()): skh_f.write( "{} {}\n".format( ",".join(name_dict.get(ssh_key_node, [ssh_key_node])), rsa_key_dict[ssh_key_node] ) ) skh_f.close() file_list.append(SSH_KNOWN_HOSTS_FILENAME) cur_inst.srv_com.set_result( "wrote {}".format(", ".join(sorted(file_list))) )
def instance_to_form_list(self, opt_ns, res_xml): def numeric_output(in_dict, state): if opt_ns.numeric: _out = "{} [{:3d}]".format( in_dict[state][0], state, ) else: _out = in_dict[state][0] return _out prc_dict = { SERVICE_OK: ("running", "ok"), SERVICE_DEAD: ("error", "critical"), SERVICE_INCOMPLETE: ("incomplete", "critical"), SERVICE_NOT_INSTALLED: ("not installed", "warning"), SERVICE_NOT_CONFIGURED: ("not configured", "warning"), } crc_dict = { CONF_STATE_RUN: ("run", "ok"), CONF_STATE_STOP: ("stop", "critical"), CONF_STATE_IP_MISMATCH: ("ip mismatch", "critical"), } meta_dict = { "t": { TARGET_STATE_RUNNING: ("run", "ok"), TARGET_STATE_STOPPED: ("stop", "critical"), }, "i": { 0: ("monitor", "ok"), 1: ("ignore", "warning"), } } if License is not None: lic_dict = { -1: ("-", ""), LicenseState.none: ("no license", "critical"), LicenseState.violated: ("parameter violated", "critical"), LicenseState.valid: ("valid", "ok"), LicenseState.grace: ("in grace", "warning"), LicenseState.expired: ("expired", "critical"), LicenseState.fp_mismatch: ("wrong fingerprint", "critical"), # LicenseState.ip_mismatch: ("ip mismatch", "critical"), } else: lic_dict = None out_bl = logging_tools.NewFormList() types = sorted(list(set(res_xml.xpath(".//instance/@runs_on", start_strings=False)))) _list = sum( [ res_xml.xpath("instance[result and @runs_on='{}']".format(_type)) for _type in types ], [] ) for act_struct in _list: _res = act_struct.find("result") p_state = int(act_struct.find(".//process_state_info").get("state", SERVICE_DEAD)) c_state = int(act_struct.find(".//configured_state_info").get("state", CONF_STATE_STOP)) if not opt_ns.failed or (opt_ns.failed and p_state not in [SERVICE_OK]): cur_line = [logging_tools.form_entry(act_struct.attrib["name"], header="Name")] cur_line.append(logging_tools.form_entry(act_struct.attrib["runs_on"], header="runson")) cur_line.append(logging_tools.form_entry(_res.find("process_state_info").get("check_source", "N/A"), header="source")) if opt_ns.process: s_info = act_struct.find(".//process_state_info") if "num_started" not in s_info.attrib: cur_line.append(logging_tools.form_entry(s_info.text)) else: num_diff, any_ok = ( int(s_info.get("num_diff")), True if int(act_struct.attrib["any-processes-ok"]) else False ) # print etree.tostring(act_struct, pretty_print=True) num_pids = len(_res.findall(".//pids/pid")) da_name = "" if any_ok: pass else: if num_diff < 0: da_name = "critical" elif num_diff > 0: da_name = "warning" cur_line.append(logging_tools.form_entry(s_info.attrib["proc_info_str"], header="Process info", display_attribute=da_name)) pid_dict = {} for cur_pid in act_struct.findall(".//pids/pid"): pid_dict[int(cur_pid.text)] = int(cur_pid.get("count", "1")) if pid_dict: p_list = sorted(pid_dict.keys()) if max(pid_dict.values()) == 1: cur_line.append(logging_tools.form_entry(logging_tools.compress_num_list(p_list), header="pids")) else: cur_line.append( logging_tools.form_entry( ",".join(["{:d}{}".format( key, " ({:d})".format(pid_dict[key]) if pid_dict[key] > 1 else "") for key in p_list] ), header="pids" ) ) else: cur_line.append(logging_tools.form_entry("no PIDs", header="pids")) if opt_ns.started: start_time = int(act_struct.find(".//process_state_info").get("start_time", "0")) if start_time: diff_time = max(0, time.mktime(time.localtime()) - start_time) diff_days = int(diff_time / (3600 * 24)) diff_hours = int((diff_time - 3600 * 24 * diff_days) / 3600) diff_mins = int((diff_time - 3600 * (24 * diff_days + diff_hours)) / 60) diff_secs = int(diff_time - 60 * (60 * (24 * diff_days + diff_hours) + diff_mins)) ret_str = "{}".format( time.strftime("%a, %d. %b %Y, %H:%M:%S", time.localtime(start_time)) ) else: ret_str = "no start info found" cur_line.append(logging_tools.form_entry(ret_str, header="started")) if opt_ns.config: cur_line.append(logging_tools.form_entry(act_struct.find(".//config_info").text, header="config info")) if opt_ns.memory: cur_mem = act_struct.find(".//memory_info") if cur_mem is not None: mem_str = process_tools.beautify_mem_info(int(cur_mem.text)) else: # no pids hence no memory info mem_str = "" cur_line.append(logging_tools.form_entry_right(mem_str, header="Memory")) if opt_ns.version: if "version" in _res.attrib: _version = _res.attrib["version"] else: _version = "" cur_line.append(logging_tools.form_entry_right(_version, header="Version")) _lic_info = _res.find("license_info") _lic_state = int(_lic_info.attrib["state"]) if lic_dict is None: cur_line.append( logging_tools.form_entry( "---", header="License", ) ) else: cur_line.append( logging_tools.form_entry( numeric_output(lic_dict, _lic_state), header="License", left=not opt_ns.numeric, display_attribute=lic_dict[_lic_state][1], ) ) cur_line.append( logging_tools.form_entry( numeric_output(prc_dict, p_state), header="PState", left=not opt_ns.numeric, display_attribute=prc_dict[p_state][1], ) ) cur_line.append( logging_tools.form_entry( numeric_output(crc_dict, c_state), header="CState", left=not opt_ns.numeric, display_attribute=crc_dict[c_state][1], ) ) if opt_ns.meta: _meta_res = act_struct.find(".//meta_result") if _meta_res is not None: t_state = int(_meta_res.get("target_state")) ignore = int(_meta_res.get("ignore")) cur_line.append( logging_tools.form_entry( numeric_output(meta_dict["t"], t_state), header="TargetState", left=not opt_ns.numeric, display_attribute=meta_dict["t"][t_state][1], ) ) cur_line.append( logging_tools.form_entry( meta_dict["i"][ignore][0], header="Ignore", display_attribute=meta_dict["i"][ignore][1], ) ) else: cur_line.append( logging_tools.form_entry( "N/A", header="TargetState", display_attribute="warning", ) ) cur_line.append( logging_tools.form_entry( "N/A", header="Ignore", display_attribute="warning", ) ) out_bl.append(cur_line) return out_bl
def interpret(self, srv_com, cur_ns): l_dict = {} for key in IPMI_LONG_LIMITS: try: l_dict[key] = float(getattr(cur_ns, key)) except: l_dict[key] = None s_list = srv_com.xpath(".//ns:sensor_list", smart_strings=False) if s_list: s_list = s_list[0] if cur_ns.arguments: el = s_list[0] cur_value = float(el.attrib["value"]) ret_state = limits.mon_STATE_OK for t_name, log, t_state in [ ("lowern", False, limits.mon_STATE_CRITICAL), ("lowerc", False, limits.mon_STATE_CRITICAL), ("lowerw", False, limits.mon_STATE_WARNING), ("upperw", True, limits.mon_STATE_WARNING), ("upperc", True, limits.mon_STATE_CRITICAL), ("uppern", True, limits.mon_STATE_CRITICAL), ]: if l_dict[t_name] is not None: if (log and cur_value >= l_dict[t_name]) or ( not log and cur_value <= l_dict[t_name]): ret_state = max(ret_state, t_state) return ret_state, "{}: {} is {:.2f} {}".format( el.attrib["key"], el.attrib["info"], cur_value, el.attrib["unit"], ) else: # list mode keys = s_list.xpath(".//@key", smart_strings=False) out_list = logging_tools.NewFormList() for key in keys: el = s_list.xpath("*[@key='{}']".format(key), smart_strings=False)[0] v_list = [ logging_tools.form_entry(key, header="key"), logging_tools.form_entry_right(el.attrib["value"], header="value"), logging_tools.form_entry_right(el.attrib["base"], header="base"), logging_tools.form_entry(el.attrib["unit"], header="unit"), logging_tools.form_entry(el.attrib["info"], header="info"), ] for l_key in IPMI_LIMITS: x_key = "limit_{}".format(l_key) v_list.append( logging_tools.form_entry(el.attrib.get(x_key, "-"), header=x_key)) out_list.append(v_list) return limits.mon_STATE_OK, "found {}:\n{}".format( logging_tools.get_plural("IPMI sensor", len(keys)), str(out_list)) else: return limits.mon_STATE_WARNING, "no IPMI sensors found"
def main(): parser = argparse.ArgumentParser() parser.add_argument("--port", type=int, default=1055, help="license server [%(default)d]") parser.add_argument("--server", type=str, default="localhost", help="license port [%(default)s]") parser.add_argument( "--license-file", type=str, default="", help= "to query multiple servers, format is {PORT@ADDR}:{PORT@ADDR} [%(default)s]" ) parser.add_argument("--mode", type=str, default="xml", choices=["xml", "check", "csv", "list"], help="output mode [%(default)s]") parser.add_argument("--check-eval", type=str, default="true", help="check string, should return true or false") opts = parser.parse_args() if opts.license_file: my_lc = sge_license_tools.LicenseCheck(license_file=opts.license_file) else: my_lc = sge_license_tools.LicenseCheck( server=opts.server, port=opts.port, ) xml_res = my_lc.check() ret_code = 0 if opts.mode == "xml": print( etree.tostring(xml_res, pretty_print=True, encoding="utf-8").decode("utf-8")) elif opts.mode == "check": glob_dict = {} for cur_lic in xml_res.findall(".//license"): lic_name = cur_lic.attrib["name"] for attr_name in ["issued", "used", "free", "reserved"]: glob_dict["{}_{}".format(lic_name, attr_name)] = int( cur_lic.attrib[attr_name]) ret_val = eval(opts.check_eval, glob_dict) if not ret_val: ret_code = 1 elif opts.mode == "csv": print(",".join(["name", "issued", "used", "free", "reserved"])) for cur_lic in xml_res.findall(".//license"): print(",".join([ cur_lic.attrib["name"], cur_lic.attrib["issued"], cur_lic.attrib["used"], cur_lic.attrib["free"], cur_lic.attrib["reserved"], ])) elif opts.mode == "list": out_form = logging_tools.NewFormList() for cur_lic in xml_res.findall(".//license"): out_form.append([ logging_tools.form_entry(cur_lic.attrib["name"], header="name"), logging_tools.form_entry_right(cur_lic.attrib["issued"], header="issued"), logging_tools.form_entry_right(cur_lic.attrib["used"], header="used"), logging_tools.form_entry_right(cur_lic.attrib["free"], header="free"), logging_tools.form_entry_right(cur_lic.attrib["reserved"], header="reserved"), ]) print(str(out_form)) sys.exit(ret_code)
def device_syslog(opt_ns, cur_dev, j_logs): print( "Information about device '{}' (full name {}, devicegroup {})".format( str(cur_dev), str(cur_dev.full_name), str(cur_dev.device_group))) print("UUID is '{}', database-ID is {:d}".format(cur_dev.uuid, cur_dev.pk)) _cr = routing.SrvTypeRouting(force=True, ignore_errors=True) _ST = "logcheck-server" if _ST in _cr.service_types: _inst_xml = InstanceXML(quiet=True) # get logcheck-server IP _ls_ip = _cr[_ST][0][1] # get logcheck-server Port _ls_port = _inst_xml.get_port_dict(_ST, ptype="command") _sc = server_command.srv_command(command="get_syslog", ) _sc["devices"] = _sc.builder( "devices", *[ _sc.builder( "device", pk="{:d}".format(cur_dev.pk), lines="{:d}".format(opt_ns.loglines), minutes="{:d}".format(opt_ns.minutes), ) ]) _conn_str = "tcp://{}:{:d}".format(_ls_ip, _ls_port) _result = net_tools.ZMQConnection("icsw_state_{:d}".format( os.getpid())).add_connection( _conn_str, _sc, ) if _result is not None: _dev = _result.xpath(".//ns:devices/ns:device[@pk]")[0] _lines = _result.xpath("ns:lines", start_el=_dev)[0] _rates = _result.xpath("ns:rates", start_el=_dev) if _rates: _rates = { int(_el.get("timeframe")): float(_el.get("rate")) for _el in _rates[0] } print("rate info: {}".format(", ".join([ "{:.2f} lines/sec in {}".format( _rates[_seconds], logging_tools.get_diff_time_str(_seconds)) for _seconds in sorted(_rates) ]))) else: print("no rate info found") print(_rates) _out_lines = logging_tools.NewFormList() for _entry in server_command.decompress(_lines.text, json=True): _out_lines.append([ logging_tools.form_entry(_entry["line_id"], header="idx"), logging_tools.form_entry( "{:04d}-{:02d}-{:02d} {:02d}:{:02d}:{:02d}".format( *_entry["line_datetime_parsed"]), header="Timestamp", ), ] + [ logging_tools.form_entry(_entry[_key], header=_key) for _key in ["hostname", "priority", "facility", "tag"] ] + [ logging_tools.form_entry(_entry["text"], header="text"), ]) print(str(_out_lines)) else: print("got no result from {} ({})".format(_conn_str, _ST)) else: print("No logcheck-server found, skipping syslog display")
def show_overview(local_mc, valid_names): mod_list = logging_tools.NewFormList() cmd_list = logging_tools.NewFormList() # iterate over modules for _idx, mod in enumerate(local_mc.module_list, 1): c_names = [name for name in valid_names if local_mc.command_dict[name].module == mod] local_valid_names = [] for com_name in c_names: local_valid_names.append(com_name) local_valid_names = sorted(local_valid_names) # show module overview mod_list.append( [ logging_tools.form_entry_right(_idx, header="#"), logging_tools.form_entry(mod.name, header="Module name"), logging_tools.form_entry(mod.Meta.uuid, header="uuid"), logging_tools.form_entry(mod.checksum, header="Checksum"), logging_tools.form_entry_center(mod.Meta.required_access.name, header="Access"), logging_tools.form_entry_center( ",".join( [ _platform.name for _platform in mod.Meta.required_platform ] ), header="Platform", ), logging_tools.form_entry_right(mod.Meta.priority, header="priority"), logging_tools.form_entry_right( "yes" if hasattr(mod, "init_machine_vector") else "no", header="MachineVector" ), logging_tools.form_entry_right(len(local_valid_names), header="#coms"), logging_tools.form_entry(", ".join(local_valid_names), header="commands"), ] ) # iterate over command for _idx, cmd_name in enumerate(sorted(local_mc.command_dict.keys()), 1): cmd = local_mc[cmd_name] # print(cmd) # print(inspect.getsource(cmd.__class__)) cmd_list.append( [ logging_tools.form_entry_right(_idx, header="#"), logging_tools.form_entry(cmd_name, header="Name"), logging_tools.form_entry(cmd.module.name, header="Module name"), logging_tools.form_entry(cmd.Meta.uuid, header="uuid"), logging_tools.form_entry(cmd.checksum, header="Checksum"), logging_tools.form_entry(cmd.Meta.check_instance.name, header="Server"), logging_tools.form_entry_center(cmd.Meta.required_access.name, header="Access"), logging_tools.form_entry_center( ",".join( [ _platform.name for _platform in cmd.Meta.required_platform ] ), header="Platform", ), logging_tools.form_entry_center( "yes" if cmd.Meta.has_perfdata else "no", header="perfdata", ), logging_tools.form_entry_center( "yes" if cmd.Meta.create_mon_check_command else "no", header="create MCC", ), logging_tools.form_entry( ", ".join( cmd.Meta.alternate_names ) if cmd.Meta.alternate_names else "---", header="Alternate names", ), logging_tools.form_entry( cmd.Meta.ports.get_port_spec(), header="PortSpec", ), logging_tools.form_entry( cmd.Meta.description, header="description", ), ] ) print("\nModule overview:\n{}".format(str(mod_list))) print("\nCommand overview:\n{}".format(str(cmd_list)))
def main(args): ignore_re = re.compile(args.ignore) coffefiles = [] htmlfiles = [] for root, dirs, files in os.walk(args.path, topdown=False): coffefiles.extend([ os.path.join(root, f) for f in files if f.endswith("coffee") and not ignore_re.search(f) ]) htmlfiles.extend([ os.path.join(root, f) for f in files if f.endswith("html") and not ignore_re.search(f) ]) print("{:d} Coffee and {:d} HTML files".format(len(coffefiles), len(htmlfiles))) def_matcher = re.compile( ".*\.(?P<type>(directive|service|controller|factory))\((\'|\")(?P<name>(.*?))(\'|\").*" ) html_matcher = re.compile( ".*script type=.text/ng-template. id=(\'|\")(?P<name>.*)(\'|\").") my_sink = DataSink(args.path) print("Getting defs...") # get definitions for name in coffefiles: for line_num, line in enumerate(open(name, "rb"), 1): match = def_matcher.match(line) if match: _gd = match.groupdict() my_sink.feed(name, line_num, line, _gd["type"], _gd["name"]) print("done (found {:d})".format(len(my_sink._defs))) # find refs in HTML to services dir_defs = my_sink.get_type_defs("directive") + my_sink.get_type_defs( "controller") dir_dict = {} for _def in dir_defs: dir_dict[_def.camel_name] = _def dir_dict[_def.hyphen_name] = _def dir_matcher = set(dir_dict.keys()) _refs = 0 s_time = time.time() for name in htmlfiles: for line_num, line in enumerate(open(name, "rb"), 1): match = html_matcher.match(line) if match: _gd = match.groupdict() my_sink.feed(name, line_num, line, "html", _gd["name"]) else: # print line _add_dict = {} for word in re.split("([^a-zA-Z\-])+", line): if word in dir_matcher: # skip lines with only closing tags if "</{}".format(word) in line and "<{}".format( word) not in line: continue # only one match per line _add_dict[word] = True for word in _add_dict.keys(): dir_dict[word].add_reference(name, line_num, line) _refs += 1 e_time = time.time() print("Reference from HTML to directive took {} (found: {:d})".format( logging_tools.get_diff_time_str(e_time - s_time), _refs, )) # find refs to Services and Factories in coffee sf_refs = my_sink.get_type_defs("factory") + my_sink.get_type_defs( "service") sf_dict = {_sf.camel_name: _sf for _sf in sf_refs} sf_matcher = set(sf_dict.keys()) # also find refs to html templates in coffee html_ref_re = re.compile( ".*(template|templateUrl)\s*:\s*.*(\'|\")(?P<temp_name>.*?)(\'|\").*") html_dict = { _html.hyphen_name: _html for _html in my_sink.get_type_defs("html") } html_matcher = set(html_dict.keys()) # print html_matcher _refs = 0 s_time = time.time() for name in coffefiles: for line_num, line in enumerate(open(name, "rb"), 1): # print line for word in re.split("([^a-zA-Z])+", line): if word in sf_matcher: # check if reference is by literal if "'{}'".format(word) in line or "\"{}\"".format( word) in line: sf_dict[word].add_reference(name, line_num, line) _refs += 1 _html_match = html_ref_re.match(line) if _html_match: _temp_ref = _html_match.groupdict()["temp_name"] if _temp_ref in html_matcher: html_dict[_temp_ref].add_reference(name, line_num, line) e_time = time.time() print("Reference from coffee to service / factory took {} (found: {:d})". format( logging_tools.get_diff_time_str(e_time - s_time), _refs, )) # generate output # raw list _list = sum( [my_sink.get_type_defs(_type) for _type in my_sink.get_types()], []) # filter if args.ignore_valid: _list = [entry for entry in _list if not entry.is_valid] name_re = re.compile(args.filter, re.IGNORECASE) _list = [entry for entry in _list if name_re.search(entry.name)] if _list: print("{} in result list:".format( logging_tools.get_plural("entry", len(_list)), )) if args.order_by == "name": _list = sorted(_list, key=attrgetter("name")) if args.order_by == "toplevel": _list = sorted(_list, key=attrgetter("top_level_dir")) out_list = logging_tools.NewFormList() files_referenced = {} for _def in _list: files_referenced.setdefault(_def.file_name, []).append(_def) out_list.append([ logging_tools.form_entry(_def.type, header="Type"), logging_tools.form_entry(_def.name, header="Name"), logging_tools.form_entry(_def.file_name, header="File"), logging_tools.form_entry_right(_def.line_num, header="line"), logging_tools.form_entry_right(len(_def.refs), header="#refs"), logging_tools.form_entry_center( "yes" if _def.namespace_ok else "no", header="NS ok"), logging_tools.form_entry_center( "yes" if _def.name_valid else "no", header="valid"), ]) if args.show_refs: out_list.extend(_def.get_ref_list()) _def.add_file_refs(files_referenced) print(str(out_list)) if args.show_refs and files_referenced: print() print("Referenced files:") print() pprint.pprint(files_referenced)
def _recv_command(self, zmq_sock): body = zmq_sock.recv().decode() if zmq_sock.getsockopt(zmq.RCVMORE): _src_id = body body = zmq_sock.recv() parameter_ok = False xml_input = body.startswith("<") if global_config["DEBUG_LEVEL"] > 3: self.log("received {:d} bytes, xml_input is {}".format(len(body), str(xml_input))) if xml_input: srv_com = server_command.srv_command(source=body) srv_com.set_result("no reply set", server_command.SRV_REPLY_STATE_UNSET) try: host = srv_com.xpath(".//ns:host", smart_strings=False)[0].text snmp_version = int(srv_com.xpath(".//ns:snmp_version", smart_strings=False)[0].text) snmp_community = srv_com.xpath(".//ns:snmp_community", smart_strings=False)[0].text comline = srv_com.xpath(".//ns:command", smart_strings=False)[0].text timeout = int(srv_com.get(".//ns:timeout", "10")) except: self._send_return(body, limits.mon_STATE_CRITICAL, "message format error: {}".format(process_tools.get_except_info())) else: envelope = srv_com["identity"].text parameter_ok = True if len(srv_com.xpath(".//ns:arg_list/text()", smart_strings=False)): comline = " ".join([comline] + srv_com.xpath(".//ns:arg_list/text()", smart_strings=False)[0].strip().split()) else: srv_com = None if body.count(";") >= 3: if body.startswith(";"): # new format proto_version, body = body[1:].split(";", 1) else: proto_version, body = ("0", body) proto_version = int(proto_version) if proto_version == 0: parts = body.split(";", 4) parts.insert(4, "10") else: parts = body.split(";", 5) envelope = parts.pop(0) # parse new format if parts[4].endswith(";"): com_part = parts[4][:-1] else: com_part = parts[4].split(";") # iterative parser try: arg_list = [] while com_part.count(";"): cur_size, cur_str = com_part.split(";", 1) cur_size = int(cur_size) com_part = cur_str[cur_size + 1:] arg_list.append(cur_str[:cur_size].decode("utf-8")) if com_part: raise ValueError("not fully parsed ({})".format(com_part)) except: self.log("error parsing {}".format(body), logging_tools.LOG_LEVEL_ERROR) arg_list = [] host, snmp_version, snmp_community, timeout = parts[0:4] timeout = int(timeout) comline = " ".join(arg_list) # print host, snmp_version, snmp_community, timeout, arg_list parameter_ok = True # envelope, host, snmp_version, snmp_community, comline = body.split(";", 4) if parameter_ok: try: snmp_version = int(snmp_version) comline_split = comline.split() scheme = comline_split.pop(0) except: self._send_return(envelope, limits.mon_STATE_CRITICAL, "message format error: {}".format(process_tools.get_except_info())) else: self.__ret_dict[envelope] = time.time() if scheme in self.__local_schemes: act_scheme, s_type = (self.__local_schemes[scheme], "L") elif scheme in self.__gen_schemes: act_handler, s_type = (self.__gen_schemes[scheme], "G") elif scheme == "list_schemes": _out_list = logging_tools.NewFormList() for _s_name in sorted(self.__local_schemes): try: _scheme = self.__local_schemes[_s_name](dummy_init=True, options="") except: pass else: _out_list.append( [ logging_tools.form_entry("local", header="type"), logging_tools.form_entry(_s_name, header="name"), logging_tools.form_entry(len(_scheme.requests), header="#reqs"), logging_tools.form_entry(", ".join(sorted([repr(_req) for _req in _scheme.requests])), header="requests"), ] ) for _s_name in sorted(self.__gen_schemes): _scheme = self.__gen_schemes[_s_name] _out_list.append( [ logging_tools.form_entry("generic", header="type"), logging_tools.form_entry(_s_name, header="name"), ] ) self._send_return(envelope, limits.mon_STATE_OK, str(_out_list)) s_type = None else: guess_list = ", ".join(difflib.get_close_matches(scheme, list(self.__local_schemes.keys()) + list(self.__gen_schemes.keys()))) err_str = "got unknown scheme '{}'{}".format( scheme, ", maybe one of {}".format(guess_list) if guess_list else ", no similar scheme found" ) self._send_return(envelope, limits.mon_STATE_CRITICAL, err_str) s_type = None if s_type: host = self._resolve_address(host) host_obj = self._get_host_object(host, snmp_community, snmp_version) if global_config["DEBUG_LEVEL"]: self.log( "got request for scheme {} (host {}, community {}, version {:d}, envelope {}, timeout {:d})".format( scheme, host, snmp_community, snmp_version, envelope, timeout, ) ) try: if s_type == "L": act_scheme = act_scheme( net_obj=host_obj, envelope=envelope, options=comline_split, xml_input=xml_input, srv_com=srv_com, init_time=time.time(), timeout=timeout, ) else: act_scheme = SNMPGeneralScheme( net_obj=host_obj, envelope=envelope, options=comline_split, xml_input=xml_input, srv_com=srv_com, init_time=time.time(), timeout=timeout, handler=act_handler, ) except IOError: err_str = "error while creating scheme {}: {}".format( scheme, process_tools.get_except_info() ) self._send_return(envelope, limits.mon_STATE_CRITICAL, err_str) else: if act_scheme.get_errors(): err_str = "problem in creating scheme {}: {}".format( scheme, ", ".join(act_scheme.get_errors()) ) self._send_return(envelope, limits.mon_STATE_CRITICAL, err_str) else: self._start_snmp_fetch(act_scheme) elif not xml_input: self._send_return(envelope, limits.mon_STATE_CRITICAL, "message format error") self.__num_messages += 1 if global_config["DEBUG_LEVEL"] > 3: self.log("recv() done") if not self.__num_messages % 100: cur_mem = process_tools.get_mem_info(self.CC.msi_block.get_unique_pids() if self.CC.msi_block else 0) self.log( "memory usage is {} after {}".format( logging_tools.get_size_str(cur_mem), logging_tools.get_plural("message", self.__num_messages) ) ) if not self.__num_messages % 50: # log process usage self.log(self.spc.get_usage())