def feed(self, port_num, result):
cur_time = time.time()
if self.__feed_time is not None:
diff_time = abs(cur_time - self.__feed_time)
self._active_keys = sorted(result.keys())
for _key in self._active_keys:
if _key not in self._speed:
_alt_key = rewrite_key(_key)
if _alt_key.count("data"):
# magic multiplicator for IB data counter
_mult = 4
unit = "Byte/s"
else:
_mult = 1
unit = "1/s"
self._mult_dict[_key] = _mult
self._speed[_key] = hm_classes.mvect_entry(
"{}.{}".format(self._get_root_key(port_num), _alt_key),
default=0.,
info="IB Readout for {} on port {}".format(
_key, port_num),
unit=unit,
base=1000,
)
# update value, update for 2 minutes
self._speed[_key].update(result[_key] * self._mult_dict[_key] /
diff_time,
valid_until=cur_time + 2 * 60)
else:
self._speed = {}
self._active_keys = []
self.__feed_time = cur_time
def _call(self, cur_time, builder):
m_vectors = []
self.__dl.check_freshness()
if self._ibq_bin:
_cmd = "{} --counters --errors --details -k -K 2>/dev/null".format(
process_tools.find_file("ibqueryerrors"))
_stat, _out = commands.getstatusoutput(_cmd)
self.ibd.feed(_out)
m_vectors = self.ibd.build_vectors(self.__dl)
else:
m_vectors = []
if False:
m_vectors.append(
E(
"machine_vector",
time="{:d}".format(int(cur_time)),
name="im",
simple="0",
uuid="5f0a0564-913a-40d1-97ee-22151ae13c7f",
))
dummy_v = hm_classes.mvect_entry("test.value",
default=0,
info="test entry",
unit="1",
base=1,
factor=1,
value=4)
m_vectors[0].append(dummy_v.build_xml(E))
# print etree.tostring(m_vectors[0], pretty_print=True)
return m_vectors
def _create_machvector(self, builder, cur_time, quota_cache):
my_vector = builder("values")
# 10 minutes valid
valid_until = cur_time + self.Meta.min_time_between_runs * 2
for dev_name, obj_type, num_id, stuff in quota_cache:
if obj_type == "group":
name = self._get_gid_info(num_id,
{}).get("groupname", "unknown")
else:
name = self._get_uid_info(num_id, {}).get("login", "unknown")
block_dict = stuff.get_block_dict()
pfix = "quota.{}.{}.{}".format(obj_type, name, dev_name)
my_vector.append(
hm_classes.mvect_entry(
"{}.soft".format(pfix),
info="Soft Limit for $2 $3 on $4",
default=0,
value=block_dict["soft"],
factor=1000,
base=1000,
valid_until=valid_until,
unit="B",
).build_xml(builder))
my_vector.append(
hm_classes.mvect_entry(
"{}.hard".format(pfix),
info="Hard Limit for $2 $3 on $4",
default=0,
value=block_dict["hard"],
factor=1000,
base=1000,
valid_until=valid_until,
unit="B",
).build_xml(builder))
my_vector.append(
hm_classes.mvect_entry(
"{}.used".format(pfix),
info="Used quota for $2 $3 on $4",
default=0,
value=block_dict["used"],
factor=1000,
base=1000,
valid_until=valid_until,
unit="B",
).build_xml(builder))
return my_vector
def _recv_vector(self, zmq_sock):
try:
rcv_com = server_command.srv_command(
source=zmq_sock.recv_unicode())
except:
self.log(
"error interpreting data as srv_command: {}".format(
process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR)
else:
for in_vector in rcv_com.xpath(".//*[@type='vector']",
smart_strings=False):
for values_list in in_vector:
for cur_value in values_list:
self.set_from_external(
hm_classes.mvect_entry(**cur_value.attrib))
self.check_timeout()
self.check_changed()
def interpret(self, srv_com, cur_ns):
if cur_ns.arguments:
re_list = [re.compile(_arg) for _arg in cur_ns.arguments]
else:
re_list = []
cur_vector = srv_com["data:machine_vector"]
if cur_ns.raw:
return limits.mon_STATE_OK, etree.tostring(
cur_vector) # @UndefinedVariable
else:
vector_keys = sorted(
srv_com.xpath(".//ns:mve/@name",
start_el=cur_vector,
smart_strings=False))
used_keys = [
key for key in vector_keys
if any([cur_re.search(key)
for cur_re in re_list]) or not re_list
]
ret_array = [
"Machinevector id {}, {}, {} shown:".format(
cur_vector.attrib["version"],
logging_tools.get_plural("key", len(vector_keys)),
logging_tools.get_plural("key", len(used_keys)),
)
]
out_list = logging_tools.new_form_list()
max_num_keys = 0
_list = []
for mv_num, mv_key in enumerate(vector_keys):
if mv_key in used_keys:
cur_xml = srv_com.xpath(
"//ns:mve[@name='{}']".format(mv_key),
start_el=cur_vector,
smart_strings=False)[0]
_mv = hm_classes.mvect_entry(cur_xml.attrib.pop("name"),
**cur_xml.attrib)
_list.append((mv_num, _mv))
max_num_keys = max(max_num_keys, _mv.num_keys)
for mv_num, entry in _list:
out_list.append(entry.get_form_entry(mv_num, max_num_keys))
ret_array.extend(unicode(out_list).split("\n"))
return limits.mon_STATE_OK, "\n".join(ret_array)
def add_ova_statistics(self, cur_time, drop_com):
_bldr = drop_com.builder
# print "*", cur_time, drop_com, _bldr
my_vector = _bldr("values")
for _csr in icswEggConsumer.objects.all():
my_vector.append(
hm_classes.mvect_entry(
"icsw.ova.{}.{}".format(_csr.content_type.model,
_csr.action),
info="Ova consumed by {} on {}".format(
_csr.action, _csr.content_type.model),
default=0,
value=_csr.consumed,
factor=1,
base=1,
valid_until=cur_time + 3600,
).build_xml(_bldr))
drop_com["vector_ova"] = my_vector
drop_com["vector_ova"].attrib["type"] = "vector"
def register_entry(self,
name,
default,
info,
unit="1",
base=1,
factor=1,
**kwargs):
# name is the key (first.second.third.fourth)
# default is a default value
# info is a description of the entry
# unit is the (SI ;-))-symbol for the entry
# base is the divider to derive the k/M/G-Values (1, 1000 or 1024)
# factor is a number the values have to be multipilicated with in order to lead to a meaningful number (for example memory or df)
self.__changed = True
self.__act_dict[name] = hm_classes.mvect_entry(name,
default=default,
info=info,
unit=unit,
base=base,
factor=factor)
def _update_fairshare(self):
self._update()
# get user list
cur_stat, cur_out = call_command(
"{} -suserl".format(
self._get_sge_bin("qconf"),
),
log_com=self.log
)
if cur_stat:
# problem calling, return immediately
return
_users = [line.strip() for line in cur_out.split("\n")]
_fs_tree = self.__sge_info.get_tree().find("fstree")
if _fs_tree is not None:
# fairshare tree found
# check if all users are present
for _user in _users:
_user_el = _fs_tree.find(".//node[@name='{}']".format(_user))
if _user_el is None:
_path = global_config["FAIRSHARE_TREE_NODE_TEMPLATE"].format(
project="defaultproject",
user=_user,
)
_shares = global_config["FAIRSHARE_TREE_DEFAULT_SHARES"]
self.log(
"No user element for user '{}' found, adding node at {} with {:d} shares".format(
_user,
_path,
_shares,
),
logging_tools.LOG_LEVEL_WARN
)
cur_stat, cur_out = call_command(
"{} -astnode {}={:d}".format(
self._get_sge_bin("qconf"),
_path,
_shares,
),
log_com=self.log
)
else:
self.log("no fairshare tree element found", logging_tools.LOG_LEVEL_WARN)
# todo: match user list with sharetree config
cur_stat, cur_out = call_command(
"{} -n -c 1 ".format(
self._get_sge_bin("sge_share_mon"),
),
log_com=self.log
)
_float_re = re.compile("^\d+\.\d+")
# headers
drop_com = server_command.srv_command(command="set_vector")
_bldr = drop_com.builder()
_rms_vector = _bldr("values")
# 10 minutes valid
act_time = int(time.time())
valid_until = act_time + 10 * 60
if not cur_stat:
for _line in cur_out.split("\n"):
_dict = {}
for _part in _line.strip().split():
_header, _value = _part.split("=", 1)
_header = _header.replace("%", "")
if _float_re.match(_value):
_dict[_header] = float(_value)
elif _value.isdigit():
_dict[_header] = int(_value)
else:
_dict[_header] = _value
# filter
if _dict["project_name"] == "defaultproject" and _dict.get("user_name", None):
_user = _dict["user_name"]
for _t_key, _key, _info in [
("cpu", "cpu", "CPU usage"),
("io", "io", "IO usage"),
("mem", "mem", "Memory usage"),
("ltcpu", "ltcpu", "long target CPU usage"),
("ltio", "ltio", "long target IO usage"),
("ltmem", "ltmem", "long target Memory usage"),
("job_count", "job_count", "Job count"),
("share.short_target", "short_target_share", "short target share"),
("share.long_target", "long_target_share", "long target share"),
("share.actual", "actual_share", "actual share"),
("shares", "shares", "configured shares"),
("level", "level", "level"),
("total", "total", "total"),
]:
_rms_vector.append(
hm_classes.mvect_entry(
"rms.fairshare.{}.{}".format(_user, _t_key),
info="{} for user {}".format(_info, _user),
default=0.,
value=_dict[_key],
factor=1,
valid_until=valid_until,
base=1000,
).build_xml(_bldr)
)
drop_com["vector_rms"] = _rms_vector
drop_com["vector_rms"].attrib["type"] = "vector"
self.vector_socket.send_unicode(unicode(drop_com))
def _generate_slotinfo(self, _res):
_queue_names = set()
_host_names = set()
act_time = int(time.time())
_s_time = time.time()
_host_stats = {}
# queue dict
_queues = {"total": QueueInfo()}
for _node in _res.findall(".//node"):
_host = _node.findtext("host")
_queue = _node.findtext("queue")
_queue_names.add(_queue)
_host_names.add(_host)
_si = _node.findtext("slot_info")
_su, _sr, _st = (int(_val) for _val in _si.split("/"))
_state = _node.findtext("state")
_queues["total"].feed(_st, _sr, _su, _state)
if _queue not in _queues:
_queues[_queue] = QueueInfo()
_queues[_queue].feed(_st, _sr, _su, _state)
if _host not in _host_stats:
_host_stats[_host] = QueueInfo()
_host_stats[_host].feed(_st, _sr, _su, _state)
# print _res
# vector socket
drop_com = server_command.srv_command(command="set_vector")
_bldr = drop_com.builder()
_rms_vector = _bldr("values")
# 10 minutes valid
valid_until = act_time + 10 * 60
_rms_vector.append(
hm_classes.mvect_entry(
"rms.clusterqueues.total",
info="ClusterQueues defined",
default=0,
value=len(_queue_names),
factor=1,
valid_until=valid_until,
base=1000,
).build_xml(_bldr)
)
_rms_vector.append(
hm_classes.mvect_entry(
"rms.hosts.total",
info="Hosts defined",
default=0,
value=len(_host_names),
factor=1,
valid_until=valid_until,
base=1000,
).build_xml(_bldr)
)
report_list = [
("total", "slots defined"),
("reserved", "slots reserved"),
("used", "slots used"),
("free", "slots free"),
("error", "instances in error state"),
("disabled", "instances in disabled state"),
("alarm", "instances in alarm state"),
("unknown", "instances in error state"),
("count", "instances"),
]
for q_name, q_value in _queues.iteritems():
# sanitize queue name
q_name = q_name.replace(".", "_")
for _key, _info in report_list:
_rms_vector.append(
hm_classes.mvect_entry(
"rms.queues.{}.{}".format(q_name, _key),
info="{} in queue {}".format(_info, q_name),
default=0,
value=getattr(q_value, _key),
factor=1,
valid_until=valid_until,
base=1000,
).build_xml(_bldr)
)
drop_com["vector_rms"] = _rms_vector
drop_com["vector_rms"].attrib["type"] = "vector"
# for cap_name in self.__cap_list:
# self.__server_cap_dict[cap_name](cur_time, drop_com)
self.vector_socket.send_unicode(unicode(drop_com))
# collectd commands
valid_hosts = {
_host: _dev for _host, _dev in [
(_host, self._get_device(_host)) for _host in _host_names
] if _dev is not None and _host in _host_stats
}
for _host_name, _dev in valid_hosts.iteritems():
mach_vect = E.machine_vector(
time="{:d}".format(act_time),
simple="0",
name=_dev.full_name,
uuid=_dev.uuid,
)
q_value = _host_stats[_host_name]
mach_vect.extend(
[
hm_classes.mvect_entry(
"rms.slots.{}".format(_key),
info="{}".format(_info),
default=0,
value=getattr(q_value, _key),
factor=1,
valid_until=valid_until,
base=1000,
).build_xml(E) for _key, _info in report_list
]
)
try:
self.collectd_socket.send_unicode(etree.tostring(mach_vect), zmq.DONTWAIT)
except:
self.log(
"error sending rms-slot info regarding {} to collectd: {}".format(
_dev.full_name,
process_tools.get_except_info(),
),
logging_tools.LOG_LEVEL_ERROR
)
_e_time = time.time()
self.log("info handling took {}".format(logging_tools.get_diff_time_str(_e_time - _s_time)))
def _update(self):
res_dict = {}
if "MD_TYPE" in global_config and global_config["MON_CURRENT_STATE"]:
_cur_time = time.time()
cur_s = LiveSocket.get_mon_live_socket()
if not self.__latest_status_query or abs(
self.__latest_status_query - _cur_time) > 10 * 60:
self.__latest_status_query = _cur_time
try:
_stat_dict = cur_s.status.call()[0]
except:
self.log(
"error getting status via livestatus: {}".format(
process_tools.get_except_info()),
logging_tools.LOG_LEVEL_ERROR)
else:
self.SH.livestatus_info(_stat_dict)
try:
result = cur_s.hosts.columns("name", "state").call()
except:
self.log(
"cannot query socket {}: {}".format(
cur_s.peer, process_tools.get_except_info()),
logging_tools.LOG_LEVEL_CRITICAL)
else:
q_list = [int(value["state"]) for value in result]
res_dict = {
s_name: q_list.count(value)
for s_name, value in [
("unknown", constants.MON_HOST_UNKNOWN),
("up", constants.MON_HOST_UP),
("down", constants.MON_HOST_DOWN),
]
}
res_dict["tot"] = sum(res_dict.values())
# cur_s.peer.close()
del cur_s
else:
self.log(
"no MD_TYPE set or MON_CURRENT_STATE is False, skipping livecheck",
logging_tools.LOG_LEVEL_WARN)
if res_dict:
self.log(
"{} status is: {:d} up, {:d} down, {:d} unknown ({:d} total)".
format(global_config["MD_TYPE"], res_dict["up"],
res_dict["down"], res_dict["unknown"], res_dict["tot"]))
drop_com = server_command.srv_command(command="set_vector")
add_obj = drop_com.builder("values")
mv_list = [
mvect_entry("mon.devices.up", info="Devices up", default=0),
mvect_entry("mon.devices.down", info="Devices down",
default=0),
mvect_entry("mon.devices.total",
info="Devices total",
default=0),
mvect_entry("mon.devices.unknown",
info="Devices unknown",
default=0),
]
cur_time = time.time()
for mv_entry, key in zip(mv_list,
["up", "down", "tot", "unknown"]):
mv_entry.update(res_dict[key])
mv_entry.valid_until = cur_time + 120
add_obj.append(mv_entry.build_xml(drop_com.builder))
drop_com["vector_loadsensor"] = add_obj
drop_com["vector_loadsensor"].attrib["type"] = "vector"
send_str = unicode(drop_com)
self.log("sending {:d} bytes to vector_socket".format(
len(send_str)))
self.vector_socket.send_unicode(send_str)
else:
self.log("empty result dict for _update()",
logging_tools.LOG_LEVEL_WARN)
def init_bg_stuff(self):
self.load_value = hm_classes.mvect_entry("sys.load1",
info="test entry",
default=0.0)
def _show_meminfo(self, res_list):
act_time = time.time()
self.__act_meminfo_line += 1
valid_entries = [
entry for entry in res_list
if entry.entry.find(".//memory_info[@valid='1']") is not None
]
act_meminfo_keys = [entry.name for entry in valid_entries]
if act_meminfo_keys != self.__last_meminfo_keys or self.__act_meminfo_line > 100:
self.__act_meminfo_line = 0
self.__last_meminfo_keys = act_meminfo_keys
self.log("Memory info mapping: {}".format(", ".join([
"{:d}: {}".format(act_meminfo_keys.index(key) + 1, key)
for key in act_meminfo_keys
])))
if hm_classes and self.vector_socket:
drop_com = server_command.srv_command(command="set_vector")
mv_valid = act_time + 2 * global_config["MIN_MEMCHECK_TIME"]
my_vector = drop_com.builder("values")
# handle removal of old keys, track pids, TODO, FIXME
old_keys = set(self.mis_dict.keys())
new_keys = set()
for entry in valid_entries:
key = entry.name
tot_mem = 0
mem_el = entry.entry.find(".//memory_info")
tot_mem = int(mem_el.text.strip())
if mem_el.find("details") is not None:
for _detail in mem_el.findall("details/mem"):
proc_name = _detail.get("name")
f_key = (key, proc_name)
info_str = "memory usage of {} ({})".format(
key, proc_name)
if f_key not in self.mis_dict:
self.mis_dict[f_key] = hm_classes.mvect_entry(
"mem.icsw.{}.{}".format(key, proc_name),
info=info_str,
default=0,
unit="Byte",
base=1024)
self.mis_dict[f_key].update(int(_detail.text))
self.mis_dict[f_key].info = info_str
self.mis_dict[f_key].valid_until = mv_valid
new_keys.add(f_key)
my_vector.append(self.mis_dict[f_key].build_xml(
drop_com.builder))
if key not in self.mis_dict:
self.mis_dict[key] = hm_classes.mvect_entry(
"mem.icsw.{}.total".format(key),
info="memory usage of {}".format(key),
default=0,
unit="Byte",
base=1024)
self.mis_dict[key].update(tot_mem)
self.mis_dict[key].valid_until = mv_valid
new_keys.add(key)
my_vector.append(self.mis_dict[key].build_xml(
drop_com.builder))
drop_com["vector"] = my_vector
drop_com["vector"].attrib["type"] = "vector"
self.vector_socket.send_unicode(unicode(drop_com))
del_keys = old_keys - new_keys
if del_keys:
self.log("removing {} from mis_dict".format(
logging_tools.get_plural("key", len(del_keys))))
for del_key in del_keys:
del self.mis_dict[del_key]
self.log("Memory info: {}".format(" / ".join([
process_tools.beautify_mem_info(int(
_el.entry.find(".//memory_info").text),
short=True)
for _el in valid_entries
])))