def __call__(self, srv_com, cur_ns):
_mounts = [
_line.strip().split()
for _line in open("/proc/mounts", "r").read().split("\n")
if _line.strip()
]
srv_com["mounts"] = server_command.compress(_mounts, json=True)
def store(
cls,
structure: object,
file_size: int,
file_type: enumerate,
install_device: object,
) -> object:
from initat.tools import server_command
_compr = server_command.compress(structure, json=True)
try:
cfile = cls.objects.get(Q(content=_compr))
except cls.DoesNotExist:
_create = True
else:
cfile.same_uploads += 1
cfile.save(update_fields=["same_uploads"])
_create = False
if _create:
cfile = BackendConfigFile(
file_size=file_size,
file_type=file_type.name,
most_recent=True,
content=_compr,
install_device=install_device,
)
cfile.save()
# remove most recent from other instances
cls.objects.filter(Q(file_type=file_type.name,
most_recent=True)).exclude(Q(
idx=cfile.idx)).update(most_recent=False)
return cfile
def distribute_info(self, srv_com, **kwargs):
di_info = server_command.decompress(srv_com["*info"], marshal=True)
self.config_store["distribute_info"] = server_command.compress(
di_info, json=True, to_string=True)
self.config_store.write()
self.SH.distribute_info(di_info)
return None
def get_pure_data(lic_content):
def _clean(_xml):
for _sig in _xml.xpath(".//icsw:signature|.//icsw:license-file-meta/icsw:creation-datetime", namespaces=ICSW_XML_NS_MAP):
_sig.text = ""
_lic_xml = etree.fromstring(server_command.decompress(lic_content)) # .encode("utf-8")))
_clean(_lic_xml)
_lic_stream = server_command.compress(etree.tostring(_lic_xml))
return _lic_stream
def __call__(self, srv_com, cur_ns):
s_time = time.time()
update_list = get_update_list()
e_time = time.time()
srv_com.set_result(
"ok got list in {}".format(
logging_tools.get_diff_time_str(e_time - s_time)), )
srv_com["update_list"] = server_command.compress(update_list,
pickle=True)
def _get_sys_info(self, *args, **kwargs):
# to get the info to the frontend
srv_com = server_command.srv_command(source=args[0])
_inst_list = list(self.__slave_configs.values())
_info_list = [_slave.info for _slave in _inst_list]
srv_com.set_result("ok set info for {}".format(
logging_tools.get_plural("system", len(_inst_list))))
srv_com["sys_info"] = server_command.compress(_info_list, json=True)
self.send_pool_message("remote_call_async_result", str(srv_com))
def _get_ref_value(in_str):
try:
_v = json.loads(in_str)
except:
# in_str has old structure, pass
# print("o", type(in_str))
return in_str
else:
# in_str is json dump, compress it
return server_command.compress(_v, json=True).decode("ascii")
def __call__(self, srv_com, cur_ns):
if PLATFORM_SYSTEM_TYPE == PlatformSystemTypeEnum.WINDOWS:
from threading import Thread
def search_func():
import win32com
import win32com.client
import pythoncom
pythoncom.CoInitialize()
update = win32com.client.Dispatch('Microsoft.Update.Session')
update_searcher = update.CreateUpdateSearcher()
search_result = update_searcher.Search(
"( IsInstalled = 0 and IsHidden = 0 )")
for i in range(search_result.Updates.Count):
title = search_result.Updates.Item(i).Title
optional = not search_result.Updates.Item(i).IsMandatory
update_list.append((title, optional))
update_list = []
update_searcher_thread = Thread(target=search_func)
update_searcher_thread.start()
update_searcher_thread.join(timeout=30 * 5)
srv_com["format"] = "windows"
srv_com["update_list"] = server_command.compress(update_list,
pickle=True)
else:
s_time = time.time()
update_list, log_list = get_update_list()
for log_entry in log_list:
self.log(log_entry)
e_time = time.time()
srv_com.set_result(
"ok got list in {}".format(
logging_tools.get_diff_time_str(e_time - s_time)), )
srv_com["format"] = "linux"
srv_com["update_list"] = server_command.compress(update_list,
pickle=True)
def __call__(self, srv_com, cur_ns):
# s_time = time.time()
if cur_ns.arguments:
name_list = cur_ns.arguments
if "cron" in name_list:
name_list.append("crond")
else:
name_list = []
_p_dict = {}
if PLATFORM_SYSTEM_TYPE == PlatformSystemTypeEnum.WINDOWS:
attr_list = [
"pid", "ppid", "name", "exe", "cmdline", "status", "ppid",
"cpu_affinity"
]
elif PLATFORM_SYSTEM_TYPE == PlatformSystemTypeEnum.LINUX:
attr_list = [
"pid",
"ppid",
"uids",
"gids",
"name",
"exe",
"cmdline",
"status",
"ppid",
"cpu_affinity",
]
else:
attr_list = ["pid", "name"]
for key, value in process_tools.get_proc_list(
proc_name_list=name_list).items():
try:
if value.is_running():
_p_dict[key] = value.as_dict(attrs=attr_list)
except psutil.NoSuchProcess:
pass
if cur_ns.arguments:
# try to be smart about cron / crond
t_dict = {
key: value
for key, value in _p_dict.items()
if value["name"] in cur_ns.arguments
}
if not t_dict and cur_ns.arguments[0] == "cron":
t_dict = {
key: value
for key, value in _p_dict.items()
if value["name"] in ["crond"]
}
_p_dict = t_dict
srv_com["process_tree"] = server_command.compress(_p_dict, json=True)
srv_com["process_tree"].attrib["format"] = "2"
def __call__(self, srv_com, cur_ns):
_stat, _out = subprocess.getstatusoutput(self.module.lsmod_command)
if _stat:
srv_com.set_result("error getting module list",
server_command.SRV_REPLY_STATE_ERROR)
else:
_lines = _out.split("\n")[1:]
srv_com["modules"] = server_command.compress(
[(_part[0], int(_part[1]), int(
_part[2]), [] if len(_part) == 3 else _part[3].split(","))
for _part in [_line.strip().split() for _line in _lines]],
json=True)
def get_syslog(srv_com):
# print srv_com.pretty_print()
for _dev in srv_com.xpath(".//ns:devices/ns:device"):
_dev.attrib.update(
{
"state": "{:d}".format(logging_tools.LOG_LEVEL_ERROR),
"result": "Device not found",
}
)
_pk = int(_dev.attrib["pk"])
if Machine.has_device(_pk):
dev = Machine.get_device(_pk)
_to_read = int(_dev.attrib.get("lines", "0"))
_minutes_to_cover = int(_dev.attrib.get("minutes", "0"))
lines = dev.filewatcher.get_logs(to_read=_to_read, minutes=_minutes_to_cover)
dev.log(
"lines found: {:d} (of {:d} / {:d})".format(
len(lines),
_to_read,
_minutes_to_cover,
)
)
_dev.attrib.update(
{
"read": "{:d}".format(len(lines)),
# for line version
"version": "1",
}
)
rates = dev.filewatcher.get_rates()
if rates:
_dev.append(
srv_com.builder(
"rates",
*[
srv_com.builder(
"rate",
timeframe="{:d}".format(_seconds),
rate="{:.4f}".format(_rate)
) for _seconds, _rate in rates.items()
]
)
)
_dev.append(
srv_com.builder(
"lines",
server_command.compress(
[_line.get_xml_format() for _line in lines],
json=True
)
)
)
def send_info_message(self):
# set latest contact for dist master
self.__latest_contact = time.time()
# send info to monitor daemon
info_list = [
_entry.get_info_dict()
for _entry in [self] + list(self.__slave_configs.values())
]
# print "ILIST", self.master_uuid, self.slave_uuid
srv_com = self._get_config_srv_command(
"info_list",
slave_info=server_command.compress(info_list, json=True),
)
self.send_to_config_server(srv_com)
def _ocsp_results(self, *args, **kwargs):
_src_proc, _src_pid, lines = args
# print "* OCSP", lines
if self.__sync_master_uuid:
self.send_command(
self.__sync_master_uuid,
server_command.srv_command(
command="ocsp_lines",
ocsp_lines=server_command.compress(lines, json=True),
))
else:
self.log(
"no sync_master_uuid set ({})".format(
logging_tools.get_plural("OCSP line", len(lines))),
logging_tools.LOG_LEVEL_ERROR)
def __call__(self, srv_com, cur_ns):
srv_com["psutil"] = "yes"
srv_com["num_cores"] = psutil.cpu_count(logical=True)
srv_com["process_tree"] = server_command.compress(
process_tools.get_proc_list(attrs=[
"pid",
"ppid",
"uids",
"gids",
"name",
"exe",
"cmdline",
"status",
"ppid",
"cpu_affinity",
]),
json=True)
def _build_file_content(self, send_list):
srv_com = self._get_slave_srv_command(
"file_content_bulk",
config_version_send="{:d}".format(self.config_version_send),
send_time="{:d}".format(int(self.send_time)),
)
_bld = srv_com.builder()
srv_com["file_list"] = _bld.file_list(*[
_bld.file(
entry["path"],
size="{:d}".format(entry["size"]),
uid="{:d}".format(entry["uid"]),
gid="{:d}".format(entry["gid"]),
) for entry in send_list
])
srv_com["bulk"] = server_command.compress("".join(
[entry["content"] for entry in send_list]).encode("utf-8"))
return srv_com
def __call__(self, srv_com, cur_ns):
installed_updates = []
if PLATFORM_SYSTEM_TYPE == PlatformSystemTypeEnum.WINDOWS:
import win32com.client
import pywintypes
update = win32com.client.Dispatch('Microsoft.Update.Session')
update_searcher = update.CreateUpdateSearcher()
count = update_searcher.GetTotalHistoryCount()
update_history = update_searcher.QueryHistory(0, count)
for i in range(update_history.Count):
update = installedupdates_command.Update()
update.title = update_history.Item(i).Title
update.date = update_history.Item(i).Date
try:
update.status = update_history.Item(i).ResultCode
except pywintypes.com_error:
update.status = "Unknown"
if update.status == 0:
update.status = "NotStarted"
elif update.status == 1:
update.status = "InProgress"
elif update.status == 2:
update.status = "Succeeded"
elif update.status == 3:
update.status = "SucceededWithErrors"
elif update.status == 4:
update.status = "Failed"
elif update.status == 5:
update.status = "Aborted"
installed_updates.append(update)
installed_updates = [(update.title, update.date.isoformat(),
update.status) for update in installed_updates]
srv_com["installed_updates"] = server_command.compress(
installed_updates, pickle=True)
def __call__(self, srv_com, cur_ns):
if os.path.isfile("/etc/debian_version"):
is_debian = True
else:
is_debian = False
rpm_root_dir, re_strs = ("/", [])
if len(cur_ns.arguments):
for arg in cur_ns.arguments:
if arg.startswith("/"):
rpm_root_dir = arg
else:
re_strs.append(arg)
if is_debian:
self.log(
"Starting dpkg -l command for root_dir '{}' ({:d} regexp_strs: {})"
.format(rpm_root_dir, len(re_strs), ", ".join(re_strs)))
else:
self.log(
"Starting rpm-list command for root_dir '{}' ({:d} regexp_strs: {})"
.format(rpm_root_dir, len(re_strs), ", ".join(re_strs)))
s_time = time.time()
log_list, ret_dict, cur_stat = rpmlist_int(rpm_root_dir, re_strs,
is_debian)
e_time = time.time()
for log in log_list:
self.log(log)
if not cur_stat:
srv_com.set_result(
"ok got list in {}".format(
logging_tools.get_diff_time_str(e_time - s_time)), )
srv_com["root_dir"] = rpm_root_dir
srv_com["format"] = "deb" if is_debian else "rpm"
srv_com["pkg_list"] = server_command.compress(ret_dict,
pickle=True)
else:
srv_com["result"].set_result(
"error getting list: {:d}".format(cur_stat),
server_command.SRV_REPLY_STATE_ERROR)
def _check_for_next_run(self):
if not self.__act_run:
self.log("init run_dict")
self.__act_run += 1
self.__act_run_dict = {
"sysinfo": process_tools.fetch_sysinfo()[1]
}
if self.__act_run not in self.__act_run_dict:
self.log("init run_dict for run %d" % (self.__act_run))
self.__act_run_dict[self.__act_run] = {
"num_threads": self.__thread_list[self.__act_run - 1],
"started": 0,
"ended": 0,
"results": {}
}
if self.__loc_config["SYNC_GLOBAL"]:
self._sync()
for num_t in range(self.__act_run_dict[self.__act_run]["num_threads"]):
self.__act_run_dict[self.__act_run]["started"] += 1
self.__stq_list[num_t].put(("start_run"))
# check for finished run
ard = self.__act_run_dict[self.__act_run]
if ard["started"] == ard["ended"]:
self.log("run %d finished" % (self.__act_run))
if self.__loc_config["SYNC_GLOBAL"]:
self._sync()
self.log("Saving result to %s" % (self.__loc_config["RESULT_FILE"]))
open(
self.__loc_config["RESULT_FILE"], "w"
).write(
server_command.compress(self.__act_run_dict, json=True)
)
if self.__act_run == self.__num_runs:
self.log("all runs finished, exiting")
self._int_error("all runs finished")
else:
self.__act_run += 1
self._check_for_next_run()
def _check_for_slaves(self, **kwargs):
master_server = device.objects.get(Q(pk=global_config["SERVER_IDX"]))
slave_servers = device.objects.exclude(
# exclude master server
Q(pk=master_server.idx)).filter(
Q(device_config__config__config_service_enum__enum_name=
icswServiceEnum.monitor_slave.name)).select_related(
"domain_tree_node")
# slave configs
self.__master_config = SyncConfig(
self,
master_server,
distributed=True if len(slave_servers) else False)
self.__slave_configs, self.__slave_lut = ({}, {})
# create lut entry for master config
self.__slave_configs[master_server.pk] = self.__master_config
self.__slave_lut[master_server.full_name] = master_server.pk
self.__slave_lut[master_server.uuid] = master_server.pk
# connect to local relayer
self.__primary_slave_uuid = routing.get_server_uuid(
icswServiceEnum.monitor_slave, master_server.uuid)
self.send_pool_message("set_sync_master_uuid",
self.__primary_slave_uuid)
self.log(" master {} (IP {}, {})".format(master_server.full_name,
"127.0.0.1",
self.__primary_slave_uuid))
self.send_pool_message("register_remote", "127.0.0.1",
self.__primary_slave_uuid,
icswServiceEnum.monitor_slave.name)
_send_data = [self.__master_config.get_send_data()]
if len(slave_servers):
self.log("found {}: {}".format(
logging_tools.get_plural("slave_server", len(slave_servers)),
", ".join(
sorted([cur_dev.full_name for cur_dev in slave_servers]))))
for cur_dev in slave_servers:
_slave_c = SyncConfig(
self,
cur_dev,
slave_name=cur_dev.full_name,
master_server=master_server,
)
self.__slave_configs[cur_dev.pk] = _slave_c
self.__slave_lut[cur_dev.full_name] = cur_dev.pk
self.__slave_lut[cur_dev.uuid] = cur_dev.pk
self.log(" slave {} (IP {}, {})".format(
_slave_c.monitor_server.full_name, _slave_c.slave_ip,
_slave_c.monitor_server.uuid))
_send_data.append(_slave_c.get_send_data())
# if _slave_c.slave_ip:
# self.send_pool_message("register_slave", _slave_c.slave_ip, _slave_c.monitor_server.uuid)
# else:
# self.log("slave has an invalid IP", logging_tools.LOG_LEVEL_CRITICAL)
else:
self.log("no slave-servers found")
# send distribution info to local syncer
distr_info = server_command.srv_command(
command="distribute_info",
info=server_command.compress(_send_data, marshal=True),
)
self.send_pool_message("distribution_info", _send_data)
self.send_sync_command(distr_info)
def _interpret(ctrl_dict, cur_ns):
# pprint.pprint(ctrl_dict)
def get_status_lines(lines):
stat_keys = [
_key for _key, _value in lines if _key.endswith("_status")
]
if stat_keys:
# status keys found, return last status
return [_value for _key, _value in lines
if _key in stat_keys][-1]
else:
# not status keys found, return first value
return lines[0][1]
def get_log_dict(lines):
return {key: value for key, value in lines}
def _get_info_str(key, lines):
_entity_type = key.split(":")[-1][0]
if _entity_type in ["v", "c", "b"]:
_ld = get_log_dict(lines)
if _entity_type == "v":
_ld["ctrl"] = key.split(":")[0]
for _key, _default in [("name", ""),
("virtual_drive", "")]:
if _key not in _ld:
_ld[_key] = _default
_info_str = "vd {virtual_drive} ('{name}', ctrl {ctrl}), RAID level {raid_level}, " \
"size={size}, drives={number_of_drives}, state={state}".format(
**_ld
)
elif _entity_type == "c":
_info_f = []
for key in ["product_name"]:
if key in _ld:
_info_f.append("{}: {}".format(
key,
_ld[key],
))
_info_str = ", ".join(_info_f)
elif _entity_type == "b":
_info_f = []
for key in [
"temperature", "voltage",
"absolute_state_of_charge",
"relative_state_of_charge",
"*learn_cycle_requested", "learn_cycle_status",
"cycle_count"
]:
if key[0] in ["*"]:
_key = key[1:]
_ignore_true = True
else:
_key = key
_ignore_true = False
if _key in _ld:
if _ignore_true and _ld[_key] is True:
continue
_info_f.append("{}: {}".format(
_key,
_ld[_key],
))
_info_str = ", ".join(_info_f)
else:
_info_str = ""
if _info_str:
_info_str = "{}: {}".format(
_expand_key(_entity_type),
_info_str,
)
return _info_str
def check_status(key, lines, check):
_entity_type = key.split(":")[-1][0]
if check == "status":
_val = [
_val for _key, _val in lines if _key.endswith("_status")
]
else:
_val = [
_val for _key, _val in lines
if _key == check or _key.replace(" ", "_") == check
]
if _val:
_key_found = True
_val = _val[0]
else:
_key_found = False
# correct key not found
_val = "not present / key not found"
# _checked not needed right now ?
_checked, _ret_state = (False, limits.mon_STATE_CRITICAL)
if _entity_type == "v":
_checked = True
if check == "state":
if _val.lower().startswith("optimal"):
_ret_state = limits.mon_STATE_OK
elif check == "current_cache_policy":
if _val.lower().strip().split()[0].startswith("writeback"):
_ret_state = limits.mon_STATE_OK
else:
_ret_state = limits.mon_STATE_WARNING
elif _entity_type == "d":
if _val.lower() == "online, spun up":
_ret_state = limits.mon_STATE_OK
elif _entity_type == "f":
if _val.lower() == "ok":
_ret_state = limits.mon_STATE_OK
elif _entity_type == "s":
if _val.lower() == "ok":
_ret_state = limits.mon_STATE_OK
elif _entity_type == "c":
_ret_state = limits.mon_STATE_OK
elif _entity_type == "b":
if _val.lower() in ("operational", "optimal"):
_ret_state = limits.mon_STATE_OK
elif not _key_found:
_ld = get_log_dict(lines)
# state not definde, check for other flags
if not _ld.get("battery_pack_missing",
True) and not _ld.get(
"battery_replacement_required", True):
_ret_state = limits.mon_STATE_OK
return _ret_state, _val, _entity_type
def get_check_list(d_type, lines):
if d_type == "virt":
_keys = [_key for _key, _value in lines]
return list(
set(_keys) & set(["state", "current_cache_policy"]))
elif d_type == "pd":
_keys = [_key for _key, _value in lines]
return ["firmware_state"]
elif d_type == "bbu":
return ["battery_state"]
elif d_type == "ctrl":
return []
else:
status = get_status_lines(lines).lower()
if status in set([
"not installed", "unknown", "medium speed",
"normal speed", "low speed", "high speed",
"not available"
]):
return None
else:
return ["status"]
def _prune(in_dict):
return {
_key: _prune(_value) if isinstance(_value, dict) else _value
for _key, _value in in_dict.items() if _value
}
def reorder_dict(in_dict):
_result = {
"c{:02d}".format(_idx): _interpret_dict("ctrl", _value)
for _idx, _value in in_dict.items()
}
# prune twice to remove empty subdicts
_result = _prune(_prune(_result))
return _result
def emit_keys(in_dict, level=0):
if isinstance(in_dict, dict):
_dk_l = set(in_dict.keys()) - {"lines", "_checks"}
r_list = sum([[
"{}{}{}".format(_s2_key, ":" if sub_key else "", sub_key)
for sub_key in emit_keys(in_dict[_s2_key], level + 1)
] for _s2_key in _dk_l], [])
# force iteration over this key (to generate info_str)
if "_checks" in in_dict:
r_list.append("")
elif not level:
# add controller keys at top level
r_list.extend(list(in_dict.keys()))
return r_list
else:
return [""]
def _interpret_dict(d_type, in_dict):
map_dict = {
"enclosures": ("e", "enclosure"),
"fans": ("f", "fan"),
"power_supplies": ("p", "psu"),
"slots": ("s", "slot"),
"temperature_senors": ("t", "tempsensor"),
"virt": ("v", "virt"),
"pd": ("d", "pd"),
"bbus": ("b", "bbu"),
}
r_dict = {}
for _key, _t in map_dict.items():
r_dict.update({
"{}{:02d}".format(_t[0], _idx):
_interpret_dict(_t[1], _value)
for _idx, _value in in_dict.get(_key, {}).items()
if isinstance(_idx, int)
})
if in_dict.get("lines", []):
r_dict["lines"] = in_dict["lines"]
_checks = get_check_list(d_type, in_dict["lines"])
if _checks:
r_dict["_checks"] = _checks
return r_dict
def get_source(_ro_dict, _key):
# return lines and check_list for given key
_res = _ro_dict
for _skey in _key.split(":"):
_res = _res[_skey]
return (_res.get("lines", []), _res.get("_checks", []))
def _expand_key(entity_type):
return {
"c": "Ctrl",
"v": "Virt",
"p": "PSU",
"s": "Slot",
"e": "Encl",
"b": "BBU",
"f": "Fan",
"d": "Disc",
}[entity_type]
def _full_key(_part):
return "{}{}".format(
{
"c": "ctrl",
"v": "virt",
"p": "psu",
"s": "slot",
"e": "encl",
"b": "bbu",
"f": "fan",
"d": "disc",
}[_part[0]],
"{:d}".format(int(_part[1:])) if len(_part) > 1 else "",
)
def get_service(_key, _check=None):
return "{}{}".format(
"/".join([_full_key(_part) for _part in _key.split(":")]),
" {}".format(_check) if _check else "",
)
def _shorten_list(in_list):
# pprint.pprint(in_list)
# shorten_re = re.compile("^(?P<pre>c\d+:(v|e)\d+:(d|s|f))(?P<rest>\d+)$")
shorten_re = re.compile(
"^(?P<pre>c\d+:((v\d+:(d|s|f))|e))(?P<rest>.*\d+)$")
_shorten_dict = {}
new_list, _shorten_keys = ([], [])
for key, _check, _info, _flag in r_list:
_keep = True
_match = shorten_re.match(key)
if _match:
_keep = False
_gd = _match.groupdict()
if _gd["pre"] not in _shorten_keys:
_shorten_keys.append(_gd["pre"])
_shorten_dict[_gd["pre"]] = {
"list": [],
"check": _check,
"infos": [],
"flag": _flag,
}
_sde = _shorten_dict[_gd["pre"]]
if (_check, _flag) == (_sde["check"], _sde["flag"]):
_sde["list"].append((key, _gd["rest"]))
_sde["infos"].append(_info)
else:
_keep = True
if _keep:
new_list.append((key, _check, _info, _flag))
for _shorten_key in _shorten_keys:
_sde = _shorten_dict[_shorten_key]
new_list.append((_shorten_key, _sde["check"],
_compress_infos(_sde["infos"]), _sde["flag"]))
# print "out"
# pprint.pprint(new_list)
# pprint.pprint(_shorten_dict)
# print "-" * 10
return new_list, _shorten_dict
def _generate_short_result(_common_key, _struct, _lss):
_state_dict = {}
# all keys are needef for the passive check result lookup key
_all_keys = []
for _state, _output, _skey in _lss:
# we ignore _info here to make things easier
_state_dict.setdefault(_state, {}).setdefault(_output,
[]).append(_skey)
_all_keys.append(_skey)
_ret_state = max(_state_dict.keys())
ret_list = []
for _state in sorted(_state_dict.keys()):
for _output in sorted(_state_dict[_state].keys()):
ret_list.append("{:d} {}: {}".format(
len(_state_dict[_state][_output]),
_output,
_compress_infos(_state_dict[_state][_output]),
))
return _compress_infos(_all_keys), _ret_state, ", ".join(ret_list)
def _compress_infos(in_list):
return logging_tools.struct_to_string(
logging_tools.list_to_struct(in_list)[0])
# rewrite bbu info
for _c_id, _c_dict in ctrl_dict.items():
if "main" in _c_dict.get("bbu_keys", {}):
_c_dict["bbus"] = {
0: {
"lines": [(_key, _value) for _key, _value in
_c_dict["bbu_keys"]["main"].items()]
}
}
del _c_dict["bbu_keys"]
if "virt" not in _c_dict:
# rewrite from old to new format
_c_dict["virt"] = {
key: {
"lines": [(line[0].lower().replace(" ", "_").replace(
"virtual_disk", "virtual_drive"), line[1])
for line in value]
}
for key, value in _c_dict["logical_lines"].items()
}
del _c_dict["logical_lines"]
# print cur_ns
# reorder dict
_ro_dict = reorder_dict(ctrl_dict)
# pprint.pprint(_ro_dict)
# pprint.pprint(ctrl_dict)
_key_list = emit_keys(_ro_dict)
# pprint.pprint(_key_list)
# print cur_ns
# interpret flags
_short_output = True if cur_ns.short_output in [
True, "1", "y", "yes", "true", "True"
] else False
_ignore_missing_bbu = True if cur_ns.ignore_missing_bbu in [
True, "1", "y", "yes", "true", "True"
] else False
_ignore_keys = [_char for _char in cur_ns.ignore_keys]
if "N" in _ignore_keys:
_ignore_keys = []
if _ignore_keys:
# filter key_list
_key_list = [
_entry for _entry in _key_list
if not any([_entry.count(_ik) for _ik in _ignore_keys])
]
if cur_ns.get_hints:
r_list = []
_ctrl_found = set()
for _key in sorted(_key_list):
_ctrl_key = _key.split(":")[0]
if _ctrl_key not in _ctrl_found:
_ctrl_found.add(_ctrl_key)
r_list.extend([
(_ctrl_key, "all", "{} info".format(_full_key(_key)),
True),
])
_lines, _checks = get_source(_ro_dict, _key)
if _checks:
if _short_output:
r_list.append((_key, "::".join(_checks),
ShortOutputKeyCache.shorten_keys([
get_service(_key, _check)
for _check in _checks
]), False))
else:
r_list.extend([(_key, _check,
get_service(_key, _check), False)
for _check in _checks])
# all checks in one line ? Todo ...
if _short_output:
# shorten list
r_list, _ignore_dict = _shorten_list(r_list)
# pprint.pprint(r_list)
return r_list
else:
_store_passive = cur_ns.passive_check_prefix != "-"
# generate passive results if cur_ns.passive_check_prefix is set (not "-")
if not _store_passive:
# only makes sense with _store_passive==True
_short_output = False
_passive_dict = {
"source": "megaraid",
"prefix": cur_ns.passive_check_prefix,
"list": [],
}
# print "*", _key_list
# if cur_ns.key != "all":
# else:
if cur_ns.check != "all":
single_key = True
_key_list = list(set(_key_list) & set([cur_ns.key]))
target_checks = set(cur_ns.check.split("::"))
# print "*", _key_list, target_checks
else:
target_checks = None
single_key = False
if cur_ns.key:
_key_list = [
_entry for _entry in _key_list
if _entry.startswith(cur_ns.key)
]
_ok_dict = {}
_ret_list = []
_g_ret_state = limits.mon_STATE_OK
# list for shortened output
r_list = []
for _key in sorted(_key_list):
# cache for short output
_so_cache = ShortOutputKeyCache()
_lines, _checks = get_source(_ro_dict, _key)
if target_checks:
_checks = list(set(_checks) & target_checks)
_info_str = _get_info_str(_key, _lines)
if not _checks:
_ret_list.append(_info_str)
for _check in _checks:
_info = get_service(_key, _check)
if _short_output:
r_list.append((_key, _check, _info, None))
_ret_state, _result, _entity_type = check_status(
_key, _lines, _check)
if _key.count(":b") and _ignore_missing_bbu:
# reduce state if necessary
_ret_state = min(_ret_state, limits.mon_STATE_WARNING)
if _store_passive and _entity_type != "c":
# never store controller checks in passive dict
if _short_output:
_so_cache.feed(_info, _ret_state, _result,
_info_str)
else:
_passive_dict["list"].append(
# format: info, ret_state, result (always show), info (only shown in case of non-OK)
(
_info,
_ret_state,
"{} {}".format(_result, _info_str),
))
else:
_ret_list.append(_info_str)
_info_str = ""
# print _info, _ret_state, _result
if _ret_state != limits.mon_STATE_OK:
_ret_list.append("{}: {}".format(_info, _result))
else:
if single_key:
_ret_list.append(_result)
if _entity_type != "c":
# we ignore contoller checks because they are only dummy checks
_ok_dict.setdefault(_entity_type, []).append(0)
_g_ret_state = max(_g_ret_state, _ret_state)
# check for addendum tio passive_dict
if _short_output and _store_passive and _so_cache.valid:
_passive_dict["list"].append(_so_cache.get_passive_entry())
_ret_list = [_val for _val in _ret_list if _val.strip()]
if _short_output:
# pprint.pprint(_passive_dict)
r_list, shorten_dict = _shorten_list(r_list)
# passive lut
_pl = {
_info: (_ret_state, _result)
for _info, _ret_state, _result in _passive_dict["list"]
}
# rewrite the passive dict
for _key, _struct in shorten_dict.items():
# pprint.pprint(_struct)
# local state list
_lss = [
list(_pl[_info]) + [_info]
for _info in _struct["infos"]
]
# remove from passive_dict.list
_passive_dict["list"] = [
(_a, _b, _c) for _a, _b, _c in _passive_dict["list"]
if _a not in _struct["infos"]
]
# add summed result
_passive_dict["list"].append(
_generate_short_result(_key, _struct, _lss))
# pprint.pprint(_passive_dict)
# pprint.pprint(_passive_dict)
if _store_passive:
ascii_chunk = server_command.compress(_passive_dict, json=True)
else:
ascii_chunk = ""
# print _ret_list, _ok_dict
if _ok_dict:
_num_ok = sum([len(_val) for _val in _ok_dict.values()])
if _num_ok == 1 and single_key:
pass
else:
_ret_list.append("{}: {}".format(
logging_tools.get_plural("OK check", _num_ok),
", ".join([
logging_tools.get_plural(_expand_key(_key),
len(_val))
for _key, _val in _ok_dict.items()
])))
# pprint.pprint(_ret_list)
return ExtReturn(_g_ret_state,
", ".join(_ret_list),
ascii_chunk=ascii_chunk)
def serialize(in_obj):
return server_command.compress(in_obj, json=True)
def entry(self, result, ref_dict):
# import pprint
# pprint.pprint(ref_dict)
m_list, gd = result
for _key in [key for key, _xml in m_list]:
# update dict with attribute dicts from the top-level node
gd.update({
_sk: _sv
for _sk, _sv in ref_dict[_key][0].items()
if _sk in ["info", "ti"]
})
# set default values
for _key, _default in [
("key", "KEY???"),
("ti", "TI???"),
]:
if _key not in gd:
gd[_key] = _default
# overrwrite empty ti
if not gd["ti"]:
gd["ti"] = gd["key"]
# expand according to dict
compound_key = self.__key.format(**gd)
_info = "{} ({:d})".format(self.__info.format(**gd), len(m_list))
# build info
_build_info = [{
"key": _s_key,
"color": _xml.attrib["color"],
"draw_type": _xml.get("draw_type", "LINE1"),
"invert": _xml.get("invert", "0"),
} for _s_key, _xml, in m_list]
_node = [{
# should not be needed for display
# "type": "compound",
"fn":
"",
"ti":
"",
"key":
compound_key,
"is_active":
True,
"is_compound":
True,
"mvvs": [{
"unit":
"",
"info":
_info,
"key":
"",
"build_info":
server_command.compress(_build_info,
json=True).decode("ascii"),
# "color": _xml.attrib["color"],
# "draw_type": _xml.get("draw_type", "LINE1"),
# "invert": _xml.get("invert", "0"),
} #
],
}]
return _node
def compress_pci_info(in_struct):
return server_command.compress(in_struct, marshal=True)
def interpret(self, srv_com, ns, *args, **kwargs):
sds = StorageDomain.deserialize(srv_com)
# print etree.tostring(sds)
ret = ExtReturn()
ret.feed_str(logging_tools.get_plural("Storagedomain", len(sds.findall(".//storage_domain"))))
ret.feed_str_state(
*SimpleCounter(
sds.xpath(".//external_status/state/text()"),
ok=["ok"],
prefix="State"
).result
)
ret.feed_str_state(
*SimpleCounter(
sds.xpath(".//storage_domain/type/text()"),
ok=["data", "export", "image", "iso"],
prefix="Domain Type"
).result
)
ret.feed_str_state(
*SimpleCounter(
sds.xpath(".//storage_domain/storage/type/text()"),
ok=["glance", "iscsi", "nfs", "fcp"],
prefix="Storage Type"
).result
)
size_dict = {
_key: sum(
[
int(_val) for _val in sds.xpath(".//storage_domain/{}/text()".format(_key))
]
) for _key in [
"used",
"available",
"committed",
]
}
size_dict["size"] = size_dict["used"] + size_dict["available"]
if ns.reference not in ["", "-"]:
_ref = server_command.decompress(ns.reference, json=True)
_passive_dict = {
"source": "ovirt_overview",
"prefix": ns.passive_check_prefix,
"list": [],
}
for run_id, run_name in zip(_ref["run_ids"], _ref["run_names"]):
_prefix = "ovirt StorageDomain {}".format(run_name)
_sd = sds.xpath(".//storage_domain[@id='{}']".format(run_id))
if len(_sd):
_sd = _sd[0]
# print(etree.tostring(_sd, pretty_print=True))
_state = _sd.findtext(".//external_status/state")
if _state is None:
# new format
_state = _sd.findtext(".//external_status")
if _state in ["ok"]:
_nag_state = limits.mon_STATE_OK
else:
_nag_state = limits.mon_STATE_CRITICAL
_stype = _sd.findtext("type")
_ret_f = [
"state is {}".format(_state),
"type is {}".format(_stype),
"storage type is {}".format(_sd.findtext("storage/type")),
]
if _stype in ["data", "iso", "export"]:
try:
_avail = int(_sd.findtext("available"))
_used = int(_sd.findtext("used"))
_committed = int(_sd.findtext("committed"))
_pused = 100. * _used / max(1, _avail + _used)
_size_str = "size is {} (used {} [{:.2f}%], avail {}), commited {}".format(
logging_tools.get_size_str(_avail + _used),
logging_tools.get_size_str(_used),
_pused,
logging_tools.get_size_str(_avail),
logging_tools.get_size_str(_committed),
)
if _pused > 95:
_nag_state = max(_nag_state, limits.mon_STATE_CRITICAL)
elif _pused > 90:
_nag_state = max(_nag_state, limits.mon_STATE_WARNING)
except:
_ret_f.append("cannot evaluate size")
_nag_state = max(_nag_state, limits.mon_STATE_WARNING)
else:
_ret_f.append(_size_str)
_passive_dict["list"].append(
(
_prefix,
_nag_state,
", ".join(_ret_f),
)
)
else:
_passive_dict["list"].append(
(
_prefix,
limits.mon_STATE_CRITICAL,
"StorageDomain not found",
)
)
ret.ascii_chunk = server_command.compress(_passive_dict, json=True)
ret.feed_str(
", ".join(
[
"{}: {}".format(
_key,
logging_tools.get_size_str(size_dict[_key])
) for _key in sorted(size_dict.keys())
]
)
)
return ret
def interpret(self, srv_com, ns, *args, **kwargs):
if ns.reference not in ["", "-"]:
# reference is a compressed dict (base64 encoded)
_ref = server_command.decompress(ns.reference, json=True)
_passive_dict = {
"source": "ovirt_overview",
"prefix": ns.passive_check_prefix,
"list": [],
}
else:
_ref = None
_passive_dict = {}
_vms = VM.deserialize(srv_com)
_num_vms = len(_vms)
_states = _vms.xpath(".//vm/status/state/text()", smart_strings=False)
_state_dict = {_state: _states.count(_state) for _state in set(_states)}
if _ref:
for run_name in _ref["run_names"]:
_vm = _vms.xpath(".//vm[name[text()='{}']]".format(run_name))
_prefix = "ovirt Domain {}".format(run_name)
if len(_vm):
_vm = _vm[0]
try:
_memory = int(_vm.findtext("memory"))
_topology = _vm.find("cpu/topology")
if "sockets" in _topology.attrib:
# old format
_sockets = int(_vm.find("cpu/topology").get("sockets"))
_cores = int(_vm.find("cpu/topology").get("cores"))
else:
# new format
_sockets = int(_topology.findtext("sockets"))
_cores = int(_topology.findtext("cores"))
_state = _vm.findtext("status/state")
if _state is None:
# try new format
_state = _vm.findtext("status")
_ret_f = [
"state is {}".format(_state),
"memory {}".format(
logging_tools.get_size_str(
_memory, long_format=True
)
),
"CPU info: {}, {}".format(
logging_tools.get_plural("socket", _sockets),
logging_tools.get_plural("core", _cores),
)
]
if _state in ["up"]:
_nag_state = limits.mon_STATE_OK
else:
_nag_state = limits.mon_STATE_CRITICAL
_passive_dict["list"].append(
(
_prefix,
_nag_state,
", ".join(_ret_f),
)
)
except:
_passive_dict["list"].append(
(
_prefix,
limits.mon_STATE_CRITICAL,
process_tools.get_except_info()
)
)
else:
_passive_dict["list"].append(
(
_prefix,
limits.mon_STATE_CRITICAL,
"domain not found",
)
)
_error_list = []
ret_state = limits.mon_STATE_OK
if _ref:
ret_state = limits.mon_STATE_OK
for _state in ["up", "down"]:
_current = _state_dict.get(_state, 0)
if _current != _ref[_state]:
_error_list.append(
"{} should by {:d} (found: {:d})".format(
_state,
_ref[_state],
_current,
)
)
ret_state = max(ret_state, limits.mon_STATE_WARNING)
if _ref is None:
ascii_chunk = ""
else:
ascii_chunk = server_command.compress(_passive_dict, json=True)
return ExtReturn(
ret_state,
"{}, {}".format(
logging_tools.get_plural("VM", _num_vms),
", ".join(
["{:d} {}".format(_state_dict[_key], _key) for _key in sorted(_state_dict)] + _error_list
),
),
ascii_chunk=ascii_chunk,
)
def send_data(self):
return server_command.compress(etree.tostring(self.xml, encoding="unicode"), json=True)
def show_hm_help(options):
from initat.host_monitoring.modules import local_mc
def dummy_print(what, log_level=logging_tools.LOG_LEVEL_OK):
print(
"{} {}".format(
logging_tools.get_log_level_str(log_level),
what
)
)
local_mc.set_log_command(dummy_print)
local_mc.build_structure()
for mod in local_mc.module_list:
# trigger checksum generation
_ = mod.checksum
for cmd_name in sorted(local_mc.command_dict.keys()):
cmd = local_mc[cmd_name]
if options.update_json:
if os.getuid() == 0 or __file__.startswith("/opt"):
raise ValueError(
"Not allowed to run update-json as root or in production environment"
)
json_file = os.path.normpath(
os.path.join(
# not beautiful but working
os.path.dirname(__file__),
"..",
"..",
"..",
"opt",
"cluster",
"share",
"json_defs",
JSON_DEFINITION_FILE
)
)
if os.path.exists(json_file):
_previous = server_command.decompress(open(json_file, "rb").read(), json=True)
else:
_previous = None
json_dump = {
"command_list": [],
"abi_version": HM_CURRENT_ABI_VERSION.name,
}
# json dump
for cmd_name in sorted(local_mc.command_dict.keys()):
cmd = local_mc[cmd_name]
if cmd.Meta.create_mon_check_command:
json_dump["command_list"].append(cmd.get_json_dump(_previous))
print("Creating definition file '{}' in {}".format(JSON_DEFINITION_FILE, json_file))
open(json_file, "wb").write(server_command.compress(json_dump, json=True))
# import pprint
# pprint.pprint(json_dump)
else:
to_show = []
for com_name in sorted(local_mc.keys()):
_show = True
if options.args:
_show = any([com_name.count(arg) for arg in options.args])
if _show:
to_show.append(com_name)
print(
"Modules defined : {:d}".format(
len(local_mc.module_list)
)
)
print(
"Commands defined / to show : {:d} / {:d}".format(
len(local_mc.command_dict),
len(to_show),
)
)
valid_names = sorted(local_mc.command_dict.keys())
if options.detail:
_sep_len = 60
for _idx, com_name in enumerate(to_show, 1):
com = local_mc.command_dict[com_name]
print(
"\n{}\n{}\n{}\n".format(
"-" * _sep_len,
"command {:3d} of {:3d}: {} (module {})".format(
_idx,
len(to_show),
com_name,
com.module.name,
),
"-" * _sep_len,
)
)
print(
"Icinga Command: {}\n".format(
com.build_icinga_command(),
)
)
com.parser.print_help()
print("\n")
if options.overview:
show_overview(local_mc, valid_names)
def __call__(self, srv_com, cur_ns):
if PLATFORM_SYSTEM_TYPE == PlatformSystemTypeEnum.WINDOWS:
uninstall_path1 = "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Uninstall"
uninstall_path2 = "SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall"
import winreg
def get_installed_packages_for_keypath(keypath):
key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, keypath, 0,
winreg.KEY_READ)
packages = []
i = 0
while True:
try:
subkey_str = winreg.EnumKey(key, i)
i += 1
subkey = winreg.OpenKey(
winreg.HKEY_LOCAL_MACHINE,
keypath + "\\" + subkey_str, 0,
winreg.KEY_READ | winreg.KEY_WOW64_64KEY)
# print subkey_str
j = 0
package = rpmlist_command.Package()
while True:
try:
subvalue = winreg.EnumValue(subkey, j)
j += 1
_val, _data, _type = subvalue
_data = str(_data).split("\\u0000")[0]
if _val == "DisplayName":
package.displayName = _data
elif _val == "DisplayVersion":
package.displayVersion = _data
elif _val == "EstimatedSize":
package.estimatedSize = _data
elif _val == "InstallDate":
package.installDate = _data
except WindowsError as e:
break
if package.displayName != "Unknown":
packages.append(package)
except WindowsError as e:
break
return packages
package_list1 = get_installed_packages_for_keypath(uninstall_path1)
package_list2 = get_installed_packages_for_keypath(uninstall_path2)
package_list1.extend(package_list2)
package_list = list(set(package_list1))
package_list.sort()
srv_com["format"] = "windows"
srv_com["pkg_list"] = server_command.compress(package_list,
pickle=True)
else:
if os.path.isfile("/etc/debian_version"):
is_debian = True
else:
is_debian = False
rpm_root_dir, re_strs = ("/", [])
arguments = getattr(cur_ns, "arguments", None)
if arguments:
for arg in cur_ns.arguments:
if arg.startswith("/"):
rpm_root_dir = arg
else:
re_strs.append(arg)
if is_debian:
self.log(
"Starting dpkg -l command for root_dir '{}' ({:d} regexp_strs{})"
.format(
rpm_root_dir,
len(re_strs),
": {}".format(", ".join(re_strs)) if re_strs else "",
))
else:
self.log(
"Starting rpm-list command for root_dir '{}' ({:d} regexp_strs{})"
.format(
rpm_root_dir,
len(re_strs),
": {}".format(", ".join(re_strs)) if re_strs else "",
))
s_time = time.time()
log_list, ret_dict, cur_stat = rpmlist_int(rpm_root_dir, re_strs,
is_debian)
e_time = time.time()
for log in log_list:
self.log(log)
if not cur_stat:
srv_com.set_result(
"ok got list in {}".format(
logging_tools.get_diff_time_str(e_time - s_time)), )
srv_com["root_dir"] = rpm_root_dir
srv_com["format"] = "deb" if is_debian else "rpm"
srv_com["pkg_list"] = server_command.compress(ret_dict,
pickle=True)
else:
srv_com["result"].set_result(
"error getting list: {:d}".format(cur_stat),
server_command.SRV_REPLY_STATE_ERROR)
def interpret(self, srv_com, ns, *args, **kwargs):
hosts = Host.deserialize(srv_com)
# print etree.tostring(hosts, pretty_print=True)
# print etree.tostring(sds)
ret = ExtReturn()
ret.feed_str(logging_tools.get_plural("Host", len(hosts.findall(".//host"))))
ret.feed_str(logging_tools.reduce_list(hosts.xpath(".//host/name/text()")))
ret.feed_str_state(
*SimpleCounter(
hosts.xpath(".//host/status/state/text()"),
ok=["up"],
prefix="State"
).result
)
ret.feed_str_state(
*SimpleCounter(
hosts.xpath(".//host/external_status/state/text()"),
ok=["ok"],
prefix="ExtStatus"
).result
)
ret.feed_str_state(
*SimpleCounter(
hosts.xpath(".//host/type/text()"),
ok=["rhel"],
prefix="Type"
).result
)
count_dict = {
_key: sum(
[
int(_val) for _val in hosts.xpath(".//host/summary/{}/text()".format(_key))
]
) for _key in [
"active",
"migrating",
"total",
]
}
if ns.reference not in ["", "-"]:
_ref = server_command.decompress(ns.reference, json=True)
_passive_dict = {
"source": "ovirt_overview",
"prefix": ns.passive_check_prefix,
"list": [],
}
for run_id, run_name in zip(_ref["run_ids"], _ref["run_names"]):
_prefix = "ovirt Host {}".format(run_name)
_host = hosts.xpath(".//host[@id='{}']".format(run_id))
if len(_host):
_host = _host[0]
_state = _host.findtext(".//status/state")
_htype = _host.findtext("type")
if _state in ["up"]:
_nag_state = limits.mon_STATE_OK
else:
_nag_state = limits.mon_STATE_CRITICAL
_ret_f = [
"state is {}".format(_state),
"type is {}".format(_htype),
]
if _host.find("summary") is not None:
_ret_f.extend(
[
"{}={:d}".format(
_key,
int(_host.findtext("summary/{}".format(_key)))
) for _key in ["active", "migrating", "total"]
]
)
if _host.find("memory") is not None:
_ret_f.append(
"mem {}".format(
logging_tools.get_size_str(int(_host.findtext("memory")))
)
)
_passive_dict["list"].append(
(
_prefix,
_nag_state,
", ".join(_ret_f),
)
)
else:
_passive_dict["list"].append(
(
_prefix,
limits.mon_STATE_CRITICAL,
"Host {} not found".format(run_name),
)
)
# print _passive_dict
ret.ascii_chunk = server_command.compress(_passive_dict, json=True)
ret.feed_str(
", ".join(
[
"{}: {}".format(
_key,
count_dict[_key]
) for _key in sorted(count_dict.keys())
]
)
)
return ret
def _get_satellite_info(self):
return self._get_dist_master_srv_command(
"satellite_info",
satellite_info=server_command.compress(self.get_satellite_info(),
json=True))
