def mi_unsub(self):
comm.execute(
"event_subscribe",
{
'event':
'E_CORE_THRESHOLD',
'socket':
'{}:{}:{}'.format(self.__rcv_proto, self.__rcv_ip,
self.__rcv_port),
'expire':
0, # there is no "event_unsubscribe", this is good enough
},
silent=True)
def __complete__(self, command, text, line, begidx, endidx):
# TODO: shall we cache this?
params_arr = comm.execute('which', {'command': command})
if len(text) == 0:
# if last character is an equal, it's probably a value, or it will
if line[-1] == "=":
return ['']
params = self.get_params_names(line)
if params is None:
flat_list = list([item for sublist in params_arr for item in sublist])
else:
# check in the line to see the parameters we've used
flat_list = set()
for p in params_arr:
sp = set(p)
if params.issubset(sp):
flat_list = flat_list.union(sp)
flat_list = flat_list - params
else:
flat_list = []
for l in params_arr:
p = [ x for x in l if x.startswith(text) ]
if len(p) != 0:
flat_list += p
l = [ x + "=" for x in list(dict.fromkeys(flat_list)) ]
return l if len(l) > 0 else ['']
def diagnose_sip(self):
# quickly ensure opensips is running
ans = comm.execute(
'get_statistics',
{'statistics': ['rcv_requests', 'rcv_replies', 'slow_messages']})
if ans is None:
return
stats = {
'ini_total':
int(ans['core:rcv_requests']) + int(ans['core:rcv_replies']),
'ini_slow':
int(ans['core:slow_messages']),
}
stats['total'] = stats['ini_total']
stats['slow'] = stats['ini_slow']
if not self.startThresholdCollector(SIP_THR_EVENTS, skip_summ=True):
return
sec = 0
try:
while True:
if not self.diagnose_sip_loop(sec, stats):
break
time.sleep(1)
sec += 1
except KeyboardInterrupt:
print('^C')
finally:
self.stopThresholdCollector()
def diagnose_db(self, dbtype, events):
# quickly ensure opensips is running
ans = comm.execute(
'get_statistics', {
'statistics': [
'{}_total_queries'.format(dbtype[0]),
'{}_slow_queries'.format(dbtype[0])
]
})
if ans is None:
return
stats = {
'ini_total':
int(ans['{}:{}_total_queries'.format(dbtype[0], dbtype[0])]),
'ini_slow':
int(ans['{}:{}_slow_queries'.format(dbtype[0], dbtype[0])]),
}
stats['total'] = stats['ini_total']
stats['slow'] = stats['ini_slow']
if not self.startThresholdCollector(events):
return
sec = 0
try:
while True:
if not self.diagnose_db_loop(sec, stats, dbtype, events):
break
time.sleep(1)
sec += 1
except KeyboardInterrupt:
print('^C')
finally:
self.stopThresholdCollector()
def diagnose_dns(self):
# quickly ensure opensips is running
ans = comm.execute(
'get_statistics',
{'statistics': ['dns_total_queries', 'dns_slow_queries']})
if ans is None:
return
stats = {
'ini_total': int(ans['dns:dns_total_queries']),
'ini_slow': int(ans['dns:dns_slow_queries']),
}
stats['total'] = stats['ini_total']
stats['slow'] = stats['ini_slow']
if not self.startThresholdCollector(DNS_THR_EVENTS):
return
sec = 0
try:
while True:
if not self.diagnose_dns_loop(sec, stats):
break
time.sleep(1)
sec += 1
except KeyboardInterrupt:
print('^C')
finally:
self.stopThresholdCollector()
def diagnose_mem_loop(self):
os.system("clear")
ans = comm.execute('get_statistics',
{'statistics': ['shmem:', 'pkmem:']})
ps = comm.execute('ps')
if ans is None or ps is None:
return False
try:
self.diagnose_shm_stats(ans)
print()
self.diagnose_pkg_stats(ans, ps)
except:
return False
self.print_diag_footer()
return True
def getOpenSIPSVersion(self):
ans = comm.execute('version')
if not ans:
return
ver = re.match(r'OpenSIPS \((?P<major>\d)\.(?P<minor>\d)\.\d.*',
ans['Server'])
return ver.groupdict()
def get_pids(self):
try:
mi_pids = comm.execute('ps')
self.pids = [str(pid['PID']) for pid in mi_pids['Processes']]
info = ["Process ID={} PID={} Type={}".
format(pid['ID'], pid['PID'], pid['Type'])
for pid in mi_pids['Processes']]
self.process_info = "\n".join(info)
except:
self.pids = []
def diagnose_db_loop(self, sec, stats, dbtype, events):
global thr_summary, thr_slowest
total_stat = '{}_total_queries'.format(dbtype[0])
slow_stat = '{}_slow_queries'.format(dbtype[0])
os.system("clear")
print("In the last {} seconds...".format(sec))
if not thr_summary:
print(" {} Queries [OK]".format(dbtype[1]))
else:
print(" {} Queries [WARNING]".format(dbtype[1]))
print(" * Slowest queries:")
for q in thr_slowest:
print(" {}: {} ({} us)".format(q[2], q[1], -q[0]))
print(" * Constantly slow queries")
for q in sorted([(v, k) for k, v in thr_summary.items()],
reverse=True)[:3]:
print(
" {}: {} ({} times exceeded threshold)".format(
q[1][1], q[1][0], q[0]))
ans = comm.execute('get_statistics',
{'statistics': [total_stat, slow_stat]})
if not ans:
return False
# was opensips restarted in the meantime? if yes, resubscribe!
if int(ans["{}:{}".format(dbtype[0], total_stat)]) < stats['total']:
stats['ini_total'] = int(ans["{}:{}".format(dbtype[0],
total_stat)])
stats['ini_slow'] = int(ans["{}:{}".format(dbtype[0], slow_stat)])
thr_summary = {}
thr_slowest = []
sec = 1
if not self.restartThresholdCollector(events):
return
stats['total'] = int(ans["{}:{}".format(dbtype[0], total_stat)]) - \
stats['ini_total']
stats['slow'] = int(ans["{}:{}".format(dbtype[0], slow_stat)]) - \
stats['ini_slow']
print(" * {} / {} queries ({}%) exceeded threshold".format(
stats['slow'], stats['total'],
int((stats['slow'] / stats['total']) * 100) \
if stats['total'] > 0 else 0))
self.print_diag_footer()
return True
def diagnose_load_loop(self, ppgroups, transports):
pgroups = ppgroups[0]
os.system("clear")
print("{}OpenSIPS Processing Status".format(25 * " "))
print()
load = comm.execute('get_statistics',
{'statistics': ['load:', 'timestamp']})
if not load:
return False
# if opensips restarted in the meantime -> refresh the proc groups
if 'ts' in pgroups and int(load['core:timestamp']) < pgroups['ts']:
pgroups = self.get_opensips_pgroups()
pgroups['ts'] = int(load['core:timestamp'])
ppgroups[0] = pgroups
else:
pgroups['ts'] = int(load['core:timestamp'])
# fetch the network waiting queues
if 'udp' in transports and pgroups['udp']:
with open('/proc/net/udp') as f:
udp_wait = [line.split() for line in f.readlines()[1:]]
self.diagnose_transport_load('udp', pgroups, load, udp_wait)
if 'tcp' in transports and pgroups['tcp']:
self.diagnose_transport_load('tcp', pgroups, load, None)
if 'hep' in transports and pgroups['hep']:
with open('/proc/net/udp') as f:
udp_wait = [line.split() for line in f.readlines()[1:]]
self.diagnose_transport_load('hep', pgroups, load, udp_wait)
print()
print(
"Info: the load percentages represent the amount of time spent by an"
)
print(
" OpenSIPS worker processing SIP messages, as opposed to waiting"
)
print(
" for new ones. The three numbers represent the 'busy' percentage"
)
print(
" over the last 1 sec, last 1 min and last 10 min, respectively."
)
self.print_diag_footer()
return True
def __complete__(self, command, text, line, begidx, endidx):
"""
helper method for autocompletion in interactive mode
"""
modules = comm.execute('pi_list')
dbs = [key for key, value in modules.items()]
if len(line.split()) < 3:
if not text:
return dbs
ret = [d for d in dbs if d.startswith(text)]
else:
if len(line.split()) == 3:
if line.split()[2] not in dbs:
if not text:
return dbs
ret = [d for d in dbs if d.startswith(text)]
else:
tables = self.get_tables(dbs)
db = line.split()[2]
if not text:
return tables[db]
ret = [t for t in tables[db] if t.startswith(text)]
else:
tables = self.get_tables(dbs)
db = line.split()[2]
if len(line.split()) == 4:
if line.split()[3] not in tables[db]:
if not text:
return tables[db]
ret = [t for t in tables[db] if t.startswith(text)]
else:
table = line.split()[3]
columns = self.get_columns(table, command)
if command != 'show':
columns = [c + '=' for c in columns]
if not text:
return columns
ret = [c for c in columns if c.startswith(text)]
else:
table = line.split()[3]
columns = self.get_columns(table,command)
if command != 'show':
columns = [c + '=' for c in columns]
if not text:
return columns
ret = [c for c in columns if c.startswith(text)]
if len(ret) == 1 and ret[0][-1] != '=':
ret[0] = ret[0] + ' '
return ret or ['']
def mi_refresh_sub(self):
now = int(time.time())
if now <= self.last_subscribe_ts + 5:
return
ans = comm.execute("event_subscribe", {
'event':
'E_CORE_THRESHOLD',
'socket':
'{}:{}:{}'.format(self.__rcv_proto, self.__rcv_ip,
self.__rcv_port),
'expire':
10,
},
silent=True)
self.last_subscribe_ts = now if ans == "OK" else 0
def diagnose_sip_loop(self, sec, stats):
global thr_slowest
os.system("clear")
print("In the last {} seconds...".format(sec))
if not thr_slowest:
print(" SIP Processing [OK]")
else:
print(" SIP Processing [WARNING]")
print(" * Slowest SIP messages:")
for q in thr_slowest:
print(" {} ({} us)".format(desc_sip_msg(q[1]),
-q[0]))
ans = comm.execute(
'get_statistics',
{'statistics': ['rcv_requests', 'rcv_replies', 'slow_messages']})
if not ans:
return False
rcv_req = int(ans["core:rcv_requests"])
rcv_rpl = int(ans["core:rcv_replies"])
slow_msgs = int(ans["core:slow_messages"])
# was opensips restarted in the meantime? if yes, resubscribe!
if rcv_req + rcv_rpl < stats['total']:
stats['ini_total'] = rcv_req + rcv_rpl
stats['ini_slow'] = slow_msgs
thr_slowest = []
sec = 1
if not self.restartThresholdCollector(SIP_THR_EVENTS,
skip_summ=True):
return
stats['total'] = rcv_req + rcv_rpl - stats['ini_total']
stats['slow'] = slow_msgs - stats['ini_slow']
print(" * {} / {} SIP messages ({}%) exceeded threshold".format(
stats['slow'], stats['total'],
int((stats['slow'] / stats['total']) * 100) \
if stats['total'] > 0 else 0))
self.print_diag_footer()
return True
def __invoke__(self, cmd, params=None):
params = self.parse_params(cmd, params)
# Mi Module works with JSON Communication
logger.debug("running command '{}' '{}'".format(cmd, params))
res = comm.execute(cmd, params)
if res is None:
return -1
output_type = cfg.get('output_type')
if output_type == "pretty-print":
self.print_pretty_print(res)
elif output_type == "dictionary":
self.print_dictionary(res)
elif output_type == "lines":
self.print_lines(res)
elif output_type == "yaml":
self.print_yaml(res)
elif output_type == "none":
pass # no one interested in the reply
else:
logger.error("unknown output_type='{}'! Dropping output!"
.format(output_type))
return 0
def get_opensips_pgroups(self):
ps = comm.execute('ps')
if ps is None:
return None
pgroups = {
'udp': {},
'tcp': {},
'hep': {},
}
for proc in ps['Processes']:
if have_psutil:
proc['cpumon'] = psutil.Process(proc['PID'])
proc['cpumon'].cpu_percent(interval=None) # begin cyle count
if proc['Type'].startswith("TCP "):
""" OpenSIPS TCP is simplified, but normalize the format"""
try:
pgroups['tcp']['TCP'].append(proc)
except:
pgroups['tcp']['TCP'] = [proc]
elif "hep_" in proc['Type']:
if proc['Type'].startswith("SIP"):
proc['Type'] = "HEP" + proc['Type'][3:]
try:
pgroups['hep'][proc['Type'][13:]].append(proc)
except:
pgroups['hep'][proc['Type'][13:]] = [proc]
elif proc['Type'].startswith("SIP receiver "):
try:
pgroups['udp'][proc['Type'][13:]].append(proc)
except:
pgroups['udp'][proc['Type'][13:]] = [proc]
return pgroups
def __get_methods__(self):
return comm.execute('which')
def do_trace(self, params):
filters = []
if params is None:
caller_f = input("Caller filter: ")
if caller_f != "":
filters.append("caller={}".format(caller_f))
callee_f = input("Callee filter: ")
if callee_f != "":
filters.append("callee={}".format(callee_f))
ip_f = input("Source IP filter: ")
if ip_f != "":
filters.append("ip={}".format(ip_f))
if len(filters) == 0:
ans = cfg.read_param(None, "No filter specified! "\
"Continue without a filter?", False, True)
if not ans:
return False
filters = None
else:
filters = params
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
trace_ip = cfg.get("trace_listen_ip")
trace_port = int(cfg.get("trace_listen_port"))
s.bind((trace_ip, trace_port))
if trace_port == 0:
trace_port = s.getsockname()[1]
s.listen(1)
conn = None
trace_name = "opensips-cli.{}".format(random.randint(0, 65536))
trace_socket = "hep:{}:{};transport=tcp;version=3".format(
trace_ip, trace_port)
args = {
'id': trace_name,
'uri': trace_socket,
}
if filters:
args['filter'] = filters
logger.debug("filters are {}".format(filters))
trace_started = comm.execute('trace_start', args)
if not trace_started:
return False
try:
conn, addr = s.accept()
logger.debug("New TCP connection from {}:{}".format(
addr[0], addr[1]))
remaining = b''
while True:
data = conn.recv(TRACE_BUFFER_SIZE)
if not data:
break
remaining = self.__print_hep(remaining + data)
if remaining is None:
break
except KeyboardInterrupt:
comm.execute('trace_stop', {'id': trace_name}, True)
if conn is not None:
conn.close()
def diagnosis_summary_loop(self):
stats = comm.execute(
'get_statistics', {
'statistics': [
'load', 'load1m', 'load10m', 'total_size',
'real_used_size', 'max_used_size', 'rcv_requests',
'rcv_replies', 'processes_number', 'slow_messages',
'pkmem:', 'dns:', 'sql:', 'cdb:'
]
})
if not stats:
return False
os.system("clear")
print("{}OpenSIPS Overview".format(" " * 25))
print("{}-----------------".format(" " * 25))
if 'load:load' in stats:
l1 = int(stats['load:load'])
l2 = int(stats['load:load1m'])
l3 = int(stats['load:load10m'])
if l1 > 20 or l2 > 20 or l3 > 20:
if l1 > 40 or l2 > 40 or l3 > 40:
if l1 > 66 or l2 > 66 or l3 > 66:
severity = "CRITICAL"
else:
severity = "WARNING"
else:
severity = "NOTICE"
else:
severity = "OK"
print("Worker Capacity: {}{}".format(severity, "" if severity == "OK" else \
" (run 'diagnose load' for more info)"))
if 'shmem:total_size' in stats:
used = int(stats['shmem:real_used_size'])
max_used = int(stats['shmem:max_used_size'])
total = int(stats['shmem:total_size'])
used_perc = round(used / total * 100)
max_used_perc = round(max_used / total * 100)
if used_perc > 70 or max_used_perc > 80:
if used_perc > 85 or max_used_perc > 90:
severity = "CRITICAL"
else:
severity = "WARNING"
else:
severity = "OK"
print("{:<16} {}{}".format("Shared Memory:", severity,
"" if severity == "OK" else \
" (run 'diagnose memory' for more info)"))
if 'load:processes_number' in stats:
procs = int(stats['load:processes_number'])
severity = "OK"
for proc in range(1, procs):
try:
used = int(stats['pkmem:{}-real_used_size'.format(proc)])
total = used + int(
stats['pkmem:{}-free_size'.format(proc)])
max_used = int(
stats['pkmem:{}-max_used_size'.format(proc)])
except:
continue
if total == 0:
continue
used_perc = round(used / total * 100)
max_used_perc = round(max_used / total * 100)
if used_perc > 70 or max_used_perc > 80:
if used_perc > 85 or max_used_perc > 90:
severity = "CRITICAL"
break
else:
severity = "WARNING"
print("{:<16} {}{}".format("Private Memory:", severity,
"" if severity == "OK" else \
" (run 'diagnose memory' for more info)"))
if 'core:slow_messages' in stats:
slow = int(stats['core:slow_messages'])
total = int(stats['core:rcv_requests']) + int(
stats['core:rcv_replies'])
try:
slow_perc = round(slow / total * 100)
except:
slow_perc = 0
if 0 <= slow_perc <= 1:
severity = "OK"
elif 2 <= slow_perc <= 5:
severity = "NOTICE"
elif 6 <= slow_perc <= 50:
severity = "WARNING"
else:
severity = "CRITICAL"
print("{:<16} {}{}".format("SIP Processing:", severity,
"" if severity == "OK" else \
" (run 'diagnose sip' for more info)"))
if 'dns:dns_slow_queries' in stats:
slow = int(stats['dns:dns_slow_queries'])
total = int(stats['dns:dns_total_queries'])
try:
slow_perc = round(slow / total * 100)
except:
slow_perc = 0
if 0 <= slow_perc <= 1:
severity = "OK"
elif 2 <= slow_perc <= 5:
severity = "NOTICE"
elif 6 <= slow_perc <= 50:
severity = "WARNING"
else:
severity = "CRITICAL"
print("{:<16} {}{}".format("DNS Queries:", severity,
"" if severity == "OK" else \
" (run 'diagnose dns' for more info)"))
if 'sql:sql_slow_queries' in stats:
slow = int(stats['sql:sql_slow_queries'])
total = int(stats['sql:sql_total_queries'])
try:
slow_perc = round(slow / total * 100)
except:
slow_perc = 0
if 0 <= slow_perc <= 1:
severity = "OK"
elif 2 <= slow_perc <= 5:
severity = "NOTICE"
elif 6 <= slow_perc <= 50:
severity = "WARNING"
else:
severity = "CRITICAL"
print("{:<16} {}{}".format("SQL queries:", severity,
"" if severity == "OK" else \
" (run 'diagnose sql' for more info)"))
if 'cdb:cdb_slow_queries' in stats:
slow = int(stats['cdb:cdb_slow_queries'])
total = int(stats['cdb:cdb_total_queries'])
try:
slow_perc = round(slow / total * 100)
except:
slow_perc = 0
if 0 <= slow_perc <= 1:
severity = "OK"
elif 2 <= slow_perc <= 5:
severity = "NOTICE"
elif 6 <= slow_perc <= 50:
severity = "WARNING"
else:
severity = "CRITICAL"
print("{:<16} {}{}".format("NoSQL Queries:", severity,
"" if severity == "OK" else \
" (run 'diagnose nosql' for more info)"))
self.print_diag_footer()
return True
