def feed(self, port_num, result): cur_time = time.time() if self.__feed_time is not None: diff_time = abs(cur_time - self.__feed_time) self._active_keys = sorted(result.keys()) for _key in self._active_keys: if _key not in self._speed: _alt_key = rewrite_key(_key) if _alt_key.count("data"): # magic multiplicator for IB data counter _mult = 4 unit = "Byte/s" else: _mult = 1 unit = "1/s" self._mult_dict[_key] = _mult self._speed[_key] = hm_classes.mvect_entry( "{}.{}".format(self._get_root_key(port_num), _alt_key), default=0., info="IB Readout for {} on port {}".format( _key, port_num), unit=unit, base=1000, ) # update value, update for 2 minutes self._speed[_key].update(result[_key] * self._mult_dict[_key] / diff_time, valid_until=cur_time + 2 * 60) else: self._speed = {} self._active_keys = [] self.__feed_time = cur_time
def _call(self, cur_time, builder): m_vectors = [] self.__dl.check_freshness() if self._ibq_bin: _cmd = "{} --counters --errors --details -k -K 2>/dev/null".format( process_tools.find_file("ibqueryerrors")) _stat, _out = commands.getstatusoutput(_cmd) self.ibd.feed(_out) m_vectors = self.ibd.build_vectors(self.__dl) else: m_vectors = [] if False: m_vectors.append( E( "machine_vector", time="{:d}".format(int(cur_time)), name="im", simple="0", uuid="5f0a0564-913a-40d1-97ee-22151ae13c7f", )) dummy_v = hm_classes.mvect_entry("test.value", default=0, info="test entry", unit="1", base=1, factor=1, value=4) m_vectors[0].append(dummy_v.build_xml(E)) # print etree.tostring(m_vectors[0], pretty_print=True) return m_vectors
def _create_machvector(self, builder, cur_time, quota_cache): my_vector = builder("values") # 10 minutes valid valid_until = cur_time + self.Meta.min_time_between_runs * 2 for dev_name, obj_type, num_id, stuff in quota_cache: if obj_type == "group": name = self._get_gid_info(num_id, {}).get("groupname", "unknown") else: name = self._get_uid_info(num_id, {}).get("login", "unknown") block_dict = stuff.get_block_dict() pfix = "quota.{}.{}.{}".format(obj_type, name, dev_name) my_vector.append( hm_classes.mvect_entry( "{}.soft".format(pfix), info="Soft Limit for $2 $3 on $4", default=0, value=block_dict["soft"], factor=1000, base=1000, valid_until=valid_until, unit="B", ).build_xml(builder)) my_vector.append( hm_classes.mvect_entry( "{}.hard".format(pfix), info="Hard Limit for $2 $3 on $4", default=0, value=block_dict["hard"], factor=1000, base=1000, valid_until=valid_until, unit="B", ).build_xml(builder)) my_vector.append( hm_classes.mvect_entry( "{}.used".format(pfix), info="Used quota for $2 $3 on $4", default=0, value=block_dict["used"], factor=1000, base=1000, valid_until=valid_until, unit="B", ).build_xml(builder)) return my_vector
def _recv_vector(self, zmq_sock): try: rcv_com = server_command.srv_command( source=zmq_sock.recv_unicode()) except: self.log( "error interpreting data as srv_command: {}".format( process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR) else: for in_vector in rcv_com.xpath(".//*[@type='vector']", smart_strings=False): for values_list in in_vector: for cur_value in values_list: self.set_from_external( hm_classes.mvect_entry(**cur_value.attrib)) self.check_timeout() self.check_changed()
def interpret(self, srv_com, cur_ns): if cur_ns.arguments: re_list = [re.compile(_arg) for _arg in cur_ns.arguments] else: re_list = [] cur_vector = srv_com["data:machine_vector"] if cur_ns.raw: return limits.mon_STATE_OK, etree.tostring( cur_vector) # @UndefinedVariable else: vector_keys = sorted( srv_com.xpath(".//ns:mve/@name", start_el=cur_vector, smart_strings=False)) used_keys = [ key for key in vector_keys if any([cur_re.search(key) for cur_re in re_list]) or not re_list ] ret_array = [ "Machinevector id {}, {}, {} shown:".format( cur_vector.attrib["version"], logging_tools.get_plural("key", len(vector_keys)), logging_tools.get_plural("key", len(used_keys)), ) ] out_list = logging_tools.new_form_list() max_num_keys = 0 _list = [] for mv_num, mv_key in enumerate(vector_keys): if mv_key in used_keys: cur_xml = srv_com.xpath( "//ns:mve[@name='{}']".format(mv_key), start_el=cur_vector, smart_strings=False)[0] _mv = hm_classes.mvect_entry(cur_xml.attrib.pop("name"), **cur_xml.attrib) _list.append((mv_num, _mv)) max_num_keys = max(max_num_keys, _mv.num_keys) for mv_num, entry in _list: out_list.append(entry.get_form_entry(mv_num, max_num_keys)) ret_array.extend(unicode(out_list).split("\n")) return limits.mon_STATE_OK, "\n".join(ret_array)
def add_ova_statistics(self, cur_time, drop_com): _bldr = drop_com.builder # print "*", cur_time, drop_com, _bldr my_vector = _bldr("values") for _csr in icswEggConsumer.objects.all(): my_vector.append( hm_classes.mvect_entry( "icsw.ova.{}.{}".format(_csr.content_type.model, _csr.action), info="Ova consumed by {} on {}".format( _csr.action, _csr.content_type.model), default=0, value=_csr.consumed, factor=1, base=1, valid_until=cur_time + 3600, ).build_xml(_bldr)) drop_com["vector_ova"] = my_vector drop_com["vector_ova"].attrib["type"] = "vector"
def register_entry(self, name, default, info, unit="1", base=1, factor=1, **kwargs): # name is the key (first.second.third.fourth) # default is a default value # info is a description of the entry # unit is the (SI ;-))-symbol for the entry # base is the divider to derive the k/M/G-Values (1, 1000 or 1024) # factor is a number the values have to be multipilicated with in order to lead to a meaningful number (for example memory or df) self.__changed = True self.__act_dict[name] = hm_classes.mvect_entry(name, default=default, info=info, unit=unit, base=base, factor=factor)
def _update_fairshare(self): self._update() # get user list cur_stat, cur_out = call_command( "{} -suserl".format( self._get_sge_bin("qconf"), ), log_com=self.log ) if cur_stat: # problem calling, return immediately return _users = [line.strip() for line in cur_out.split("\n")] _fs_tree = self.__sge_info.get_tree().find("fstree") if _fs_tree is not None: # fairshare tree found # check if all users are present for _user in _users: _user_el = _fs_tree.find(".//node[@name='{}']".format(_user)) if _user_el is None: _path = global_config["FAIRSHARE_TREE_NODE_TEMPLATE"].format( project="defaultproject", user=_user, ) _shares = global_config["FAIRSHARE_TREE_DEFAULT_SHARES"] self.log( "No user element for user '{}' found, adding node at {} with {:d} shares".format( _user, _path, _shares, ), logging_tools.LOG_LEVEL_WARN ) cur_stat, cur_out = call_command( "{} -astnode {}={:d}".format( self._get_sge_bin("qconf"), _path, _shares, ), log_com=self.log ) else: self.log("no fairshare tree element found", logging_tools.LOG_LEVEL_WARN) # todo: match user list with sharetree config cur_stat, cur_out = call_command( "{} -n -c 1 ".format( self._get_sge_bin("sge_share_mon"), ), log_com=self.log ) _float_re = re.compile("^\d+\.\d+") # headers drop_com = server_command.srv_command(command="set_vector") _bldr = drop_com.builder() _rms_vector = _bldr("values") # 10 minutes valid act_time = int(time.time()) valid_until = act_time + 10 * 60 if not cur_stat: for _line in cur_out.split("\n"): _dict = {} for _part in _line.strip().split(): _header, _value = _part.split("=", 1) _header = _header.replace("%", "") if _float_re.match(_value): _dict[_header] = float(_value) elif _value.isdigit(): _dict[_header] = int(_value) else: _dict[_header] = _value # filter if _dict["project_name"] == "defaultproject" and _dict.get("user_name", None): _user = _dict["user_name"] for _t_key, _key, _info in [ ("cpu", "cpu", "CPU usage"), ("io", "io", "IO usage"), ("mem", "mem", "Memory usage"), ("ltcpu", "ltcpu", "long target CPU usage"), ("ltio", "ltio", "long target IO usage"), ("ltmem", "ltmem", "long target Memory usage"), ("job_count", "job_count", "Job count"), ("share.short_target", "short_target_share", "short target share"), ("share.long_target", "long_target_share", "long target share"), ("share.actual", "actual_share", "actual share"), ("shares", "shares", "configured shares"), ("level", "level", "level"), ("total", "total", "total"), ]: _rms_vector.append( hm_classes.mvect_entry( "rms.fairshare.{}.{}".format(_user, _t_key), info="{} for user {}".format(_info, _user), default=0., value=_dict[_key], factor=1, valid_until=valid_until, base=1000, ).build_xml(_bldr) ) drop_com["vector_rms"] = _rms_vector drop_com["vector_rms"].attrib["type"] = "vector" self.vector_socket.send_unicode(unicode(drop_com))
def _generate_slotinfo(self, _res): _queue_names = set() _host_names = set() act_time = int(time.time()) _s_time = time.time() _host_stats = {} # queue dict _queues = {"total": QueueInfo()} for _node in _res.findall(".//node"): _host = _node.findtext("host") _queue = _node.findtext("queue") _queue_names.add(_queue) _host_names.add(_host) _si = _node.findtext("slot_info") _su, _sr, _st = (int(_val) for _val in _si.split("/")) _state = _node.findtext("state") _queues["total"].feed(_st, _sr, _su, _state) if _queue not in _queues: _queues[_queue] = QueueInfo() _queues[_queue].feed(_st, _sr, _su, _state) if _host not in _host_stats: _host_stats[_host] = QueueInfo() _host_stats[_host].feed(_st, _sr, _su, _state) # print _res # vector socket drop_com = server_command.srv_command(command="set_vector") _bldr = drop_com.builder() _rms_vector = _bldr("values") # 10 minutes valid valid_until = act_time + 10 * 60 _rms_vector.append( hm_classes.mvect_entry( "rms.clusterqueues.total", info="ClusterQueues defined", default=0, value=len(_queue_names), factor=1, valid_until=valid_until, base=1000, ).build_xml(_bldr) ) _rms_vector.append( hm_classes.mvect_entry( "rms.hosts.total", info="Hosts defined", default=0, value=len(_host_names), factor=1, valid_until=valid_until, base=1000, ).build_xml(_bldr) ) report_list = [ ("total", "slots defined"), ("reserved", "slots reserved"), ("used", "slots used"), ("free", "slots free"), ("error", "instances in error state"), ("disabled", "instances in disabled state"), ("alarm", "instances in alarm state"), ("unknown", "instances in error state"), ("count", "instances"), ] for q_name, q_value in _queues.iteritems(): # sanitize queue name q_name = q_name.replace(".", "_") for _key, _info in report_list: _rms_vector.append( hm_classes.mvect_entry( "rms.queues.{}.{}".format(q_name, _key), info="{} in queue {}".format(_info, q_name), default=0, value=getattr(q_value, _key), factor=1, valid_until=valid_until, base=1000, ).build_xml(_bldr) ) drop_com["vector_rms"] = _rms_vector drop_com["vector_rms"].attrib["type"] = "vector" # for cap_name in self.__cap_list: # self.__server_cap_dict[cap_name](cur_time, drop_com) self.vector_socket.send_unicode(unicode(drop_com)) # collectd commands valid_hosts = { _host: _dev for _host, _dev in [ (_host, self._get_device(_host)) for _host in _host_names ] if _dev is not None and _host in _host_stats } for _host_name, _dev in valid_hosts.iteritems(): mach_vect = E.machine_vector( time="{:d}".format(act_time), simple="0", name=_dev.full_name, uuid=_dev.uuid, ) q_value = _host_stats[_host_name] mach_vect.extend( [ hm_classes.mvect_entry( "rms.slots.{}".format(_key), info="{}".format(_info), default=0, value=getattr(q_value, _key), factor=1, valid_until=valid_until, base=1000, ).build_xml(E) for _key, _info in report_list ] ) try: self.collectd_socket.send_unicode(etree.tostring(mach_vect), zmq.DONTWAIT) except: self.log( "error sending rms-slot info regarding {} to collectd: {}".format( _dev.full_name, process_tools.get_except_info(), ), logging_tools.LOG_LEVEL_ERROR ) _e_time = time.time() self.log("info handling took {}".format(logging_tools.get_diff_time_str(_e_time - _s_time)))
def _update(self): res_dict = {} if "MD_TYPE" in global_config and global_config["MON_CURRENT_STATE"]: _cur_time = time.time() cur_s = LiveSocket.get_mon_live_socket() if not self.__latest_status_query or abs( self.__latest_status_query - _cur_time) > 10 * 60: self.__latest_status_query = _cur_time try: _stat_dict = cur_s.status.call()[0] except: self.log( "error getting status via livestatus: {}".format( process_tools.get_except_info()), logging_tools.LOG_LEVEL_ERROR) else: self.SH.livestatus_info(_stat_dict) try: result = cur_s.hosts.columns("name", "state").call() except: self.log( "cannot query socket {}: {}".format( cur_s.peer, process_tools.get_except_info()), logging_tools.LOG_LEVEL_CRITICAL) else: q_list = [int(value["state"]) for value in result] res_dict = { s_name: q_list.count(value) for s_name, value in [ ("unknown", constants.MON_HOST_UNKNOWN), ("up", constants.MON_HOST_UP), ("down", constants.MON_HOST_DOWN), ] } res_dict["tot"] = sum(res_dict.values()) # cur_s.peer.close() del cur_s else: self.log( "no MD_TYPE set or MON_CURRENT_STATE is False, skipping livecheck", logging_tools.LOG_LEVEL_WARN) if res_dict: self.log( "{} status is: {:d} up, {:d} down, {:d} unknown ({:d} total)". format(global_config["MD_TYPE"], res_dict["up"], res_dict["down"], res_dict["unknown"], res_dict["tot"])) drop_com = server_command.srv_command(command="set_vector") add_obj = drop_com.builder("values") mv_list = [ mvect_entry("mon.devices.up", info="Devices up", default=0), mvect_entry("mon.devices.down", info="Devices down", default=0), mvect_entry("mon.devices.total", info="Devices total", default=0), mvect_entry("mon.devices.unknown", info="Devices unknown", default=0), ] cur_time = time.time() for mv_entry, key in zip(mv_list, ["up", "down", "tot", "unknown"]): mv_entry.update(res_dict[key]) mv_entry.valid_until = cur_time + 120 add_obj.append(mv_entry.build_xml(drop_com.builder)) drop_com["vector_loadsensor"] = add_obj drop_com["vector_loadsensor"].attrib["type"] = "vector" send_str = unicode(drop_com) self.log("sending {:d} bytes to vector_socket".format( len(send_str))) self.vector_socket.send_unicode(send_str) else: self.log("empty result dict for _update()", logging_tools.LOG_LEVEL_WARN)
def init_bg_stuff(self): self.load_value = hm_classes.mvect_entry("sys.load1", info="test entry", default=0.0)
def _show_meminfo(self, res_list): act_time = time.time() self.__act_meminfo_line += 1 valid_entries = [ entry for entry in res_list if entry.entry.find(".//memory_info[@valid='1']") is not None ] act_meminfo_keys = [entry.name for entry in valid_entries] if act_meminfo_keys != self.__last_meminfo_keys or self.__act_meminfo_line > 100: self.__act_meminfo_line = 0 self.__last_meminfo_keys = act_meminfo_keys self.log("Memory info mapping: {}".format(", ".join([ "{:d}: {}".format(act_meminfo_keys.index(key) + 1, key) for key in act_meminfo_keys ]))) if hm_classes and self.vector_socket: drop_com = server_command.srv_command(command="set_vector") mv_valid = act_time + 2 * global_config["MIN_MEMCHECK_TIME"] my_vector = drop_com.builder("values") # handle removal of old keys, track pids, TODO, FIXME old_keys = set(self.mis_dict.keys()) new_keys = set() for entry in valid_entries: key = entry.name tot_mem = 0 mem_el = entry.entry.find(".//memory_info") tot_mem = int(mem_el.text.strip()) if mem_el.find("details") is not None: for _detail in mem_el.findall("details/mem"): proc_name = _detail.get("name") f_key = (key, proc_name) info_str = "memory usage of {} ({})".format( key, proc_name) if f_key not in self.mis_dict: self.mis_dict[f_key] = hm_classes.mvect_entry( "mem.icsw.{}.{}".format(key, proc_name), info=info_str, default=0, unit="Byte", base=1024) self.mis_dict[f_key].update(int(_detail.text)) self.mis_dict[f_key].info = info_str self.mis_dict[f_key].valid_until = mv_valid new_keys.add(f_key) my_vector.append(self.mis_dict[f_key].build_xml( drop_com.builder)) if key not in self.mis_dict: self.mis_dict[key] = hm_classes.mvect_entry( "mem.icsw.{}.total".format(key), info="memory usage of {}".format(key), default=0, unit="Byte", base=1024) self.mis_dict[key].update(tot_mem) self.mis_dict[key].valid_until = mv_valid new_keys.add(key) my_vector.append(self.mis_dict[key].build_xml( drop_com.builder)) drop_com["vector"] = my_vector drop_com["vector"].attrib["type"] = "vector" self.vector_socket.send_unicode(unicode(drop_com)) del_keys = old_keys - new_keys if del_keys: self.log("removing {} from mis_dict".format( logging_tools.get_plural("key", len(del_keys)))) for del_key in del_keys: del self.mis_dict[del_key] self.log("Memory info: {}".format(" / ".join([ process_tools.beautify_mem_info(int( _el.entry.find(".//memory_info").text), short=True) for _el in valid_entries ])))