def mi_unsub(self): comm.execute( "event_subscribe", { 'event': 'E_CORE_THRESHOLD', 'socket': '{}:{}:{}'.format(self.__rcv_proto, self.__rcv_ip, self.__rcv_port), 'expire': 0, # there is no "event_unsubscribe", this is good enough }, silent=True)
def __complete__(self, command, text, line, begidx, endidx): # TODO: shall we cache this? params_arr = comm.execute('which', {'command': command}) if len(text) == 0: # if last character is an equal, it's probably a value, or it will if line[-1] == "=": return [''] params = self.get_params_names(line) if params is None: flat_list = list([item for sublist in params_arr for item in sublist]) else: # check in the line to see the parameters we've used flat_list = set() for p in params_arr: sp = set(p) if params.issubset(sp): flat_list = flat_list.union(sp) flat_list = flat_list - params else: flat_list = [] for l in params_arr: p = [ x for x in l if x.startswith(text) ] if len(p) != 0: flat_list += p l = [ x + "=" for x in list(dict.fromkeys(flat_list)) ] return l if len(l) > 0 else ['']
def diagnose_sip(self): # quickly ensure opensips is running ans = comm.execute( 'get_statistics', {'statistics': ['rcv_requests', 'rcv_replies', 'slow_messages']}) if ans is None: return stats = { 'ini_total': int(ans['core:rcv_requests']) + int(ans['core:rcv_replies']), 'ini_slow': int(ans['core:slow_messages']), } stats['total'] = stats['ini_total'] stats['slow'] = stats['ini_slow'] if not self.startThresholdCollector(SIP_THR_EVENTS, skip_summ=True): return sec = 0 try: while True: if not self.diagnose_sip_loop(sec, stats): break time.sleep(1) sec += 1 except KeyboardInterrupt: print('^C') finally: self.stopThresholdCollector()
def diagnose_db(self, dbtype, events): # quickly ensure opensips is running ans = comm.execute( 'get_statistics', { 'statistics': [ '{}_total_queries'.format(dbtype[0]), '{}_slow_queries'.format(dbtype[0]) ] }) if ans is None: return stats = { 'ini_total': int(ans['{}:{}_total_queries'.format(dbtype[0], dbtype[0])]), 'ini_slow': int(ans['{}:{}_slow_queries'.format(dbtype[0], dbtype[0])]), } stats['total'] = stats['ini_total'] stats['slow'] = stats['ini_slow'] if not self.startThresholdCollector(events): return sec = 0 try: while True: if not self.diagnose_db_loop(sec, stats, dbtype, events): break time.sleep(1) sec += 1 except KeyboardInterrupt: print('^C') finally: self.stopThresholdCollector()
def diagnose_dns(self): # quickly ensure opensips is running ans = comm.execute( 'get_statistics', {'statistics': ['dns_total_queries', 'dns_slow_queries']}) if ans is None: return stats = { 'ini_total': int(ans['dns:dns_total_queries']), 'ini_slow': int(ans['dns:dns_slow_queries']), } stats['total'] = stats['ini_total'] stats['slow'] = stats['ini_slow'] if not self.startThresholdCollector(DNS_THR_EVENTS): return sec = 0 try: while True: if not self.diagnose_dns_loop(sec, stats): break time.sleep(1) sec += 1 except KeyboardInterrupt: print('^C') finally: self.stopThresholdCollector()
def diagnose_mem_loop(self): os.system("clear") ans = comm.execute('get_statistics', {'statistics': ['shmem:', 'pkmem:']}) ps = comm.execute('ps') if ans is None or ps is None: return False try: self.diagnose_shm_stats(ans) print() self.diagnose_pkg_stats(ans, ps) except: return False self.print_diag_footer() return True
def getOpenSIPSVersion(self): ans = comm.execute('version') if not ans: return ver = re.match(r'OpenSIPS \((?P<major>\d)\.(?P<minor>\d)\.\d.*', ans['Server']) return ver.groupdict()
def get_pids(self): try: mi_pids = comm.execute('ps') self.pids = [str(pid['PID']) for pid in mi_pids['Processes']] info = ["Process ID={} PID={} Type={}". format(pid['ID'], pid['PID'], pid['Type']) for pid in mi_pids['Processes']] self.process_info = "\n".join(info) except: self.pids = []
def diagnose_db_loop(self, sec, stats, dbtype, events): global thr_summary, thr_slowest total_stat = '{}_total_queries'.format(dbtype[0]) slow_stat = '{}_slow_queries'.format(dbtype[0]) os.system("clear") print("In the last {} seconds...".format(sec)) if not thr_summary: print(" {} Queries [OK]".format(dbtype[1])) else: print(" {} Queries [WARNING]".format(dbtype[1])) print(" * Slowest queries:") for q in thr_slowest: print(" {}: {} ({} us)".format(q[2], q[1], -q[0])) print(" * Constantly slow queries") for q in sorted([(v, k) for k, v in thr_summary.items()], reverse=True)[:3]: print( " {}: {} ({} times exceeded threshold)".format( q[1][1], q[1][0], q[0])) ans = comm.execute('get_statistics', {'statistics': [total_stat, slow_stat]}) if not ans: return False # was opensips restarted in the meantime? if yes, resubscribe! if int(ans["{}:{}".format(dbtype[0], total_stat)]) < stats['total']: stats['ini_total'] = int(ans["{}:{}".format(dbtype[0], total_stat)]) stats['ini_slow'] = int(ans["{}:{}".format(dbtype[0], slow_stat)]) thr_summary = {} thr_slowest = [] sec = 1 if not self.restartThresholdCollector(events): return stats['total'] = int(ans["{}:{}".format(dbtype[0], total_stat)]) - \ stats['ini_total'] stats['slow'] = int(ans["{}:{}".format(dbtype[0], slow_stat)]) - \ stats['ini_slow'] print(" * {} / {} queries ({}%) exceeded threshold".format( stats['slow'], stats['total'], int((stats['slow'] / stats['total']) * 100) \ if stats['total'] > 0 else 0)) self.print_diag_footer() return True
def diagnose_load_loop(self, ppgroups, transports): pgroups = ppgroups[0] os.system("clear") print("{}OpenSIPS Processing Status".format(25 * " ")) print() load = comm.execute('get_statistics', {'statistics': ['load:', 'timestamp']}) if not load: return False # if opensips restarted in the meantime -> refresh the proc groups if 'ts' in pgroups and int(load['core:timestamp']) < pgroups['ts']: pgroups = self.get_opensips_pgroups() pgroups['ts'] = int(load['core:timestamp']) ppgroups[0] = pgroups else: pgroups['ts'] = int(load['core:timestamp']) # fetch the network waiting queues if 'udp' in transports and pgroups['udp']: with open('/proc/net/udp') as f: udp_wait = [line.split() for line in f.readlines()[1:]] self.diagnose_transport_load('udp', pgroups, load, udp_wait) if 'tcp' in transports and pgroups['tcp']: self.diagnose_transport_load('tcp', pgroups, load, None) if 'hep' in transports and pgroups['hep']: with open('/proc/net/udp') as f: udp_wait = [line.split() for line in f.readlines()[1:]] self.diagnose_transport_load('hep', pgroups, load, udp_wait) print() print( "Info: the load percentages represent the amount of time spent by an" ) print( " OpenSIPS worker processing SIP messages, as opposed to waiting" ) print( " for new ones. The three numbers represent the 'busy' percentage" ) print( " over the last 1 sec, last 1 min and last 10 min, respectively." ) self.print_diag_footer() return True
def __complete__(self, command, text, line, begidx, endidx): """ helper method for autocompletion in interactive mode """ modules = comm.execute('pi_list') dbs = [key for key, value in modules.items()] if len(line.split()) < 3: if not text: return dbs ret = [d for d in dbs if d.startswith(text)] else: if len(line.split()) == 3: if line.split()[2] not in dbs: if not text: return dbs ret = [d for d in dbs if d.startswith(text)] else: tables = self.get_tables(dbs) db = line.split()[2] if not text: return tables[db] ret = [t for t in tables[db] if t.startswith(text)] else: tables = self.get_tables(dbs) db = line.split()[2] if len(line.split()) == 4: if line.split()[3] not in tables[db]: if not text: return tables[db] ret = [t for t in tables[db] if t.startswith(text)] else: table = line.split()[3] columns = self.get_columns(table, command) if command != 'show': columns = [c + '=' for c in columns] if not text: return columns ret = [c for c in columns if c.startswith(text)] else: table = line.split()[3] columns = self.get_columns(table,command) if command != 'show': columns = [c + '=' for c in columns] if not text: return columns ret = [c for c in columns if c.startswith(text)] if len(ret) == 1 and ret[0][-1] != '=': ret[0] = ret[0] + ' ' return ret or ['']
def mi_refresh_sub(self): now = int(time.time()) if now <= self.last_subscribe_ts + 5: return ans = comm.execute("event_subscribe", { 'event': 'E_CORE_THRESHOLD', 'socket': '{}:{}:{}'.format(self.__rcv_proto, self.__rcv_ip, self.__rcv_port), 'expire': 10, }, silent=True) self.last_subscribe_ts = now if ans == "OK" else 0
def diagnose_sip_loop(self, sec, stats): global thr_slowest os.system("clear") print("In the last {} seconds...".format(sec)) if not thr_slowest: print(" SIP Processing [OK]") else: print(" SIP Processing [WARNING]") print(" * Slowest SIP messages:") for q in thr_slowest: print(" {} ({} us)".format(desc_sip_msg(q[1]), -q[0])) ans = comm.execute( 'get_statistics', {'statistics': ['rcv_requests', 'rcv_replies', 'slow_messages']}) if not ans: return False rcv_req = int(ans["core:rcv_requests"]) rcv_rpl = int(ans["core:rcv_replies"]) slow_msgs = int(ans["core:slow_messages"]) # was opensips restarted in the meantime? if yes, resubscribe! if rcv_req + rcv_rpl < stats['total']: stats['ini_total'] = rcv_req + rcv_rpl stats['ini_slow'] = slow_msgs thr_slowest = [] sec = 1 if not self.restartThresholdCollector(SIP_THR_EVENTS, skip_summ=True): return stats['total'] = rcv_req + rcv_rpl - stats['ini_total'] stats['slow'] = slow_msgs - stats['ini_slow'] print(" * {} / {} SIP messages ({}%) exceeded threshold".format( stats['slow'], stats['total'], int((stats['slow'] / stats['total']) * 100) \ if stats['total'] > 0 else 0)) self.print_diag_footer() return True
def __invoke__(self, cmd, params=None): params = self.parse_params(cmd, params) # Mi Module works with JSON Communication logger.debug("running command '{}' '{}'".format(cmd, params)) res = comm.execute(cmd, params) if res is None: return -1 output_type = cfg.get('output_type') if output_type == "pretty-print": self.print_pretty_print(res) elif output_type == "dictionary": self.print_dictionary(res) elif output_type == "lines": self.print_lines(res) elif output_type == "yaml": self.print_yaml(res) elif output_type == "none": pass # no one interested in the reply else: logger.error("unknown output_type='{}'! Dropping output!" .format(output_type)) return 0
def get_opensips_pgroups(self): ps = comm.execute('ps') if ps is None: return None pgroups = { 'udp': {}, 'tcp': {}, 'hep': {}, } for proc in ps['Processes']: if have_psutil: proc['cpumon'] = psutil.Process(proc['PID']) proc['cpumon'].cpu_percent(interval=None) # begin cyle count if proc['Type'].startswith("TCP "): """ OpenSIPS TCP is simplified, but normalize the format""" try: pgroups['tcp']['TCP'].append(proc) except: pgroups['tcp']['TCP'] = [proc] elif "hep_" in proc['Type']: if proc['Type'].startswith("SIP"): proc['Type'] = "HEP" + proc['Type'][3:] try: pgroups['hep'][proc['Type'][13:]].append(proc) except: pgroups['hep'][proc['Type'][13:]] = [proc] elif proc['Type'].startswith("SIP receiver "): try: pgroups['udp'][proc['Type'][13:]].append(proc) except: pgroups['udp'][proc['Type'][13:]] = [proc] return pgroups
def __get_methods__(self): return comm.execute('which')
def do_trace(self, params): filters = [] if params is None: caller_f = input("Caller filter: ") if caller_f != "": filters.append("caller={}".format(caller_f)) callee_f = input("Callee filter: ") if callee_f != "": filters.append("callee={}".format(callee_f)) ip_f = input("Source IP filter: ") if ip_f != "": filters.append("ip={}".format(ip_f)) if len(filters) == 0: ans = cfg.read_param(None, "No filter specified! "\ "Continue without a filter?", False, True) if not ans: return False filters = None else: filters = params s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) trace_ip = cfg.get("trace_listen_ip") trace_port = int(cfg.get("trace_listen_port")) s.bind((trace_ip, trace_port)) if trace_port == 0: trace_port = s.getsockname()[1] s.listen(1) conn = None trace_name = "opensips-cli.{}".format(random.randint(0, 65536)) trace_socket = "hep:{}:{};transport=tcp;version=3".format( trace_ip, trace_port) args = { 'id': trace_name, 'uri': trace_socket, } if filters: args['filter'] = filters logger.debug("filters are {}".format(filters)) trace_started = comm.execute('trace_start', args) if not trace_started: return False try: conn, addr = s.accept() logger.debug("New TCP connection from {}:{}".format( addr[0], addr[1])) remaining = b'' while True: data = conn.recv(TRACE_BUFFER_SIZE) if not data: break remaining = self.__print_hep(remaining + data) if remaining is None: break except KeyboardInterrupt: comm.execute('trace_stop', {'id': trace_name}, True) if conn is not None: conn.close()
def diagnosis_summary_loop(self): stats = comm.execute( 'get_statistics', { 'statistics': [ 'load', 'load1m', 'load10m', 'total_size', 'real_used_size', 'max_used_size', 'rcv_requests', 'rcv_replies', 'processes_number', 'slow_messages', 'pkmem:', 'dns:', 'sql:', 'cdb:' ] }) if not stats: return False os.system("clear") print("{}OpenSIPS Overview".format(" " * 25)) print("{}-----------------".format(" " * 25)) if 'load:load' in stats: l1 = int(stats['load:load']) l2 = int(stats['load:load1m']) l3 = int(stats['load:load10m']) if l1 > 20 or l2 > 20 or l3 > 20: if l1 > 40 or l2 > 40 or l3 > 40: if l1 > 66 or l2 > 66 or l3 > 66: severity = "CRITICAL" else: severity = "WARNING" else: severity = "NOTICE" else: severity = "OK" print("Worker Capacity: {}{}".format(severity, "" if severity == "OK" else \ " (run 'diagnose load' for more info)")) if 'shmem:total_size' in stats: used = int(stats['shmem:real_used_size']) max_used = int(stats['shmem:max_used_size']) total = int(stats['shmem:total_size']) used_perc = round(used / total * 100) max_used_perc = round(max_used / total * 100) if used_perc > 70 or max_used_perc > 80: if used_perc > 85 or max_used_perc > 90: severity = "CRITICAL" else: severity = "WARNING" else: severity = "OK" print("{:<16} {}{}".format("Shared Memory:", severity, "" if severity == "OK" else \ " (run 'diagnose memory' for more info)")) if 'load:processes_number' in stats: procs = int(stats['load:processes_number']) severity = "OK" for proc in range(1, procs): try: used = int(stats['pkmem:{}-real_used_size'.format(proc)]) total = used + int( stats['pkmem:{}-free_size'.format(proc)]) max_used = int( stats['pkmem:{}-max_used_size'.format(proc)]) except: continue if total == 0: continue used_perc = round(used / total * 100) max_used_perc = round(max_used / total * 100) if used_perc > 70 or max_used_perc > 80: if used_perc > 85 or max_used_perc > 90: severity = "CRITICAL" break else: severity = "WARNING" print("{:<16} {}{}".format("Private Memory:", severity, "" if severity == "OK" else \ " (run 'diagnose memory' for more info)")) if 'core:slow_messages' in stats: slow = int(stats['core:slow_messages']) total = int(stats['core:rcv_requests']) + int( stats['core:rcv_replies']) try: slow_perc = round(slow / total * 100) except: slow_perc = 0 if 0 <= slow_perc <= 1: severity = "OK" elif 2 <= slow_perc <= 5: severity = "NOTICE" elif 6 <= slow_perc <= 50: severity = "WARNING" else: severity = "CRITICAL" print("{:<16} {}{}".format("SIP Processing:", severity, "" if severity == "OK" else \ " (run 'diagnose sip' for more info)")) if 'dns:dns_slow_queries' in stats: slow = int(stats['dns:dns_slow_queries']) total = int(stats['dns:dns_total_queries']) try: slow_perc = round(slow / total * 100) except: slow_perc = 0 if 0 <= slow_perc <= 1: severity = "OK" elif 2 <= slow_perc <= 5: severity = "NOTICE" elif 6 <= slow_perc <= 50: severity = "WARNING" else: severity = "CRITICAL" print("{:<16} {}{}".format("DNS Queries:", severity, "" if severity == "OK" else \ " (run 'diagnose dns' for more info)")) if 'sql:sql_slow_queries' in stats: slow = int(stats['sql:sql_slow_queries']) total = int(stats['sql:sql_total_queries']) try: slow_perc = round(slow / total * 100) except: slow_perc = 0 if 0 <= slow_perc <= 1: severity = "OK" elif 2 <= slow_perc <= 5: severity = "NOTICE" elif 6 <= slow_perc <= 50: severity = "WARNING" else: severity = "CRITICAL" print("{:<16} {}{}".format("SQL queries:", severity, "" if severity == "OK" else \ " (run 'diagnose sql' for more info)")) if 'cdb:cdb_slow_queries' in stats: slow = int(stats['cdb:cdb_slow_queries']) total = int(stats['cdb:cdb_total_queries']) try: slow_perc = round(slow / total * 100) except: slow_perc = 0 if 0 <= slow_perc <= 1: severity = "OK" elif 2 <= slow_perc <= 5: severity = "NOTICE" elif 6 <= slow_perc <= 50: severity = "WARNING" else: severity = "CRITICAL" print("{:<16} {}{}".format("NoSQL Queries:", severity, "" if severity == "OK" else \ " (run 'diagnose nosql' for more info)")) self.print_diag_footer() return True