def get(self):
host = self.config.get('destination', self.hostname)
out, err, code = yield fork('/bin/ping',
args=('-q', '-n', '-c', '5', '-i', '0.2',
host),
timeout=30.0)
if code == 0:
# Successful ping
try:
out = out.strip('\n').split('\n')[-2:]
loss = int(out[0].split()[5].strip('%'))
stat = out[1].split()[3].split('/')
pmin, avg, pmax, mdev = [float(i) for i in stat]
event = [
self.createEvent('ok',
'Latency to %s' % host,
avg,
prefix="latency"),
self.createEvent('ok',
'%s%% loss to %s' % (loss, host),
loss,
prefix="loss"),
]
except Exception, e:
print("Could not parse response %s" % repr(out))
event = None
def get(self):
disks = self.config.get('disks')
out, err, code = yield fork('/bin/df', args=('-lPx', 'tmpfs',))
out = [i.split() for i in out.strip('\n').split('\n')[1:]]
events = []
for disk, size, used, free, util, mount in out:
if disks and (disk not in disks):
continue
if disk != "udev":
util = int(util.strip('%'))
used = int(used)
free = int(free)
events.extend([
self.createEvent('ok', 'Disk usage %s%%' % (util),
util, prefix=disk),
self.createEvent('ok', 'Disk usage %s kB' % (used),
used, prefix="%s.bytes" % disk),
self.createEvent('ok', 'Disk free %s kB' % (free),
free, prefix="%s.free" % disk)
])
defer.returnValue(events)
def get(self):
out, err, code = yield fork('/bin/ps', args=('-e', ))
count = len(out.strip('\n').split('\n')) - 1
defer.returnValue(
self.createEvent('ok', 'Process count %s' % (count), count))
def get(self):
host = self.config.get('destination', self.hostname)
t0 = time.time()
try:
out, err, code = yield fork('/usr/bin/avprobe',
args=('rtsp://%s/sample_100kbit.mp4' % host, ), timeout=30.0)
except:
code = 1
err = None
t_delta = (time.time() - t0) * 1000
if code == 0:
e = self.createEvent('ok', 'RTSP Request time to %s' % host,
t_delta)
else:
if err:
try:
error = err.strip('\n').split('\n')[-2]
except:
error = err.replace('\n', '-')
else:
error = "Execution error"
e = self.createEvent('critical',
'Unable to stream %s:%s' % (host, error),
t_delta)
defer.returnValue(e)
def get(self):
host = self.config.get('destination', self.hostname)
t0 = time.time()
try:
out, err, code = yield fork('/usr/bin/avprobe',
args=('rtsp://%s/sample_100kbit.mp4' % host, ), timeout=30.0)
except:
code = 1
err = None
t_delta = (time.time() - t0) * 1000
if code == 0:
e = self.createEvent('ok', 'RTSP Request time to %s' % host,
t_delta)
else:
if err:
try:
error = err.strip('\n').split('\n')[-2]
except:
error = err.replace('\n', '-')
else:
error = "Execution error"
e = self.createEvent('critical',
'Unable to stream %s:%s' % (host, error),
t_delta)
defer.returnValue(e)
def _get_uc_stats(self):
out, err, code = yield fork(self.uc, args=('stats', ))
if code == 0:
defer.returnValue(out.strip('\n').split('\n'))
else:
log.msg('Error running unbound-control: ' + repr(err))
defer.returnValue([])
def _get_smart(self, device):
out, err, code = yield fork('/usr/sbin/smartctl',
args=('-A', device))
if code == 0:
defer.returnValue(out.strip('\n').split('\n'))
else:
defer.returnValue([])
def get(self):
out, err, code = yield fork('/bin/ps', args=('-e',))
count = len(out.strip('\n').split('\n')) - 1
defer.returnValue(
self.createEvent('ok', 'Process count %s' % (count), count)
)
def test_utils_fork_timeout(self):
died = False
try:
o, e, c = yield fork('sleep', args=('2',), timeout=0.1)
except:
died = True
self.assertTrue(died)
def test_utils_fork_timeout(self):
died = False
try:
o, e, c = yield fork('sleep', args=('2',), timeout=0.1)
except error.ProcessTerminated:
died = True
self.assertTrue(died)
def getVideo(self, url, out, time=10):
try:
if os.path.exists(out):
os.unlink(out)
out, err, code = yield utils.fork('/usr/bin/avconv',
args=('-i', url, '-t', str(time),
'-c', 'copy', out),
timeout=60.0)
except Exception, e:
out, err, code = "", str(e), 1
def get(self):
out, err, code = yield fork('/usr/bin/sudo',
args=('ipsec', 'statusall'))
connections = {}
s = 0
for l in out.strip('\n').split('\n'):
if l == "Connections:":
s = 1
continue
elif l == "Routed Connections:":
s = 2
elif "Security Associations" in l:
s = 3
elif l[0] == ' ' and ':' in l:
if s == 1:
con, detail = l.strip().split(': ', 1)
detail = detail.strip()
if con not in connections:
connections[con] = {
'source': detail.split('...')[0],
'destination': detail.split('...')[1].split()[0],
'up': False
}
elif s == 3:
con, detail = l.strip().split(': ', 1)
detail = detail.strip()
if '[' in con:
con = con.split('[')[0]
else:
con = con.split('{')[0]
if 'ESTABLISHED' in detail:
connections[con]['up'] = True
events = []
for k, v in connections.items():
if v['up']:
events.append(
self.createEvent('ok',
'IPSec tunnel %s up' % k,
1,
prefix=k))
else:
events.append(
self.createEvent('critical',
'IPSec tunnel %s down' % k,
0,
prefix=k))
defer.returnValue(events)
def _get_disks(self):
out, err, code = yield fork('/usr/sbin/smartctl',
args=('--scan',))
if code != 0:
defer.returnValue([])
out = out.strip('\n').split('\n')
devices = []
for ln in out:
if '/dev' in ln:
devices.append(ln.split()[0])
defer.returnValue(devices)
def get(self):
out, err, code = yield fork('/bin/df', args=('-lPx','tmpfs',))
out = [i.split() for i in out.strip('\n').split('\n')[1:]]
events = []
for disk, size, used, free, util, mount in out:
if disk != "udev":
util = int(util.strip('%'))
events.append(
self.createEvent('ok', 'Disk usage %s' % (util),
util, prefix=disk)
)
defer.returnValue(events)
def get(self):
out, err, code = yield fork("/usr/bin/sudo", args=("ipsec", "statusall"))
connections = {}
s = 0
for l in out.strip("\n").split("\n"):
if l == "Connections:":
s = 1
continue
elif l == "Routed Connections:":
s = 2
elif "Security Associations" in l:
s = 3
elif l[0] == " " and ":" in l:
if s == 1:
con, detail = l.strip().split(": ", 1)
detail = detail.strip()
if con not in connections:
connections[con] = {
"source": detail.split("...")[0],
"destination": detail.split("...")[1].split()[0],
"up": False,
}
elif s == 3:
con, detail = l.strip().split(": ", 1)
detail = detail.strip()
if "[" in con:
con = con.split("[")[0]
else:
con = con.split("{")[0]
if "ESTABLISHED" in detail:
connections[con]["up"] = True
events = []
for k, v in connections.items():
if v["up"]:
events.append(self.createEvent("ok", "IPSec tunnel %s up" % k, 1, prefix=k))
else:
events.append(self.createEvent("critical", "IPSec tunnel %s down" % k, 0, prefix=k))
defer.returnValue(events)
def get(self):
out, err, code = yield fork('/bin/df', args=(
'-lPx',
'tmpfs',
))
out = [i.split() for i in out.strip('\n').split('\n')[1:]]
events = []
for disk, size, used, free, util, mount in out:
if disk != "udev":
util = int(util.strip('%'))
events.append(
self.createEvent('ok',
'Disk usage %s' % (util),
util,
prefix=disk))
defer.returnValue(events)
def get(self):
out, err, code = yield fork(self.clipath,
args=('-n', str(self.db), 'llen', self.queue,)
)
events = []
if code == 0:
val = int(out.strip('\n').split()[-1])
defer.returnValue([
self.createEvent('ok', '%s queue length' % self.queue, val),
self.createEvent('ok', 'Queue rate', val, prefix='rate',
aggregation=Counter)
])
else:
err = 'Error running %s: %s' % (self.clipath, repr(err))
log.msg(err)
defer.returnValue(
self.createEvent('critical', err, None)
)
def get(self):
host = self.config.get('destination', self.hostname)
out, err, code = yield fork('/bin/ping',
args=('-q', '-n', '-c', '5', '-i', '0.2', host), timeout=30.0)
if code == 0:
# Successful ping
try:
out = out.strip('\n').split('\n')[-2:]
loss = int(out[0].split()[5].strip('%'))
stat = out[1].split()[3].split('/')
pmin, avg, pmax, mdev = [float(i) for i in stat]
event = [
self.createEvent('ok', 'Latency to %s' % host, avg,
prefix="latency"),
self.createEvent('ok', '%s%% loss to %s' % (loss,host), loss,
prefix="loss"),
]
except Exception, e:
print("Could not parse response %s" % repr(out))
event = None
def get(self):
disks = self.config.get('disks')
out, err, code = yield fork('/bin/df', args=(
'-lPx',
'tmpfs',
))
out = [i.split() for i in out.strip('\n').split('\n')[1:]]
events = []
for disk, size, used, free, util, mount in out:
if disks and (disk not in disks):
continue
if disk != "udev":
util = int(util.strip('%'))
used = int(used)
free = int(free)
events.extend([
self.createEvent('ok',
'Disk usage %s%%' % (util),
util,
prefix=disk),
self.createEvent('ok',
'Disk usage %s kB' % (used),
used,
prefix="%s.bytes" % disk),
self.createEvent('ok',
'Disk free %s kB' % (free),
free,
prefix="%s.free" % disk)
])
defer.returnValue(events)
def get(self):
vhost = self.config.get('vhost', '/')
mqctl = self.config.get('rabbitmqctl', '/usr/sbin/rabbitmqctl')
out, err, code = yield fork(mqctl, args=(
'list_queues', '-p', vhost, 'name', 'messages_ready',
'messages_unacknowledged'
))
if code == 0:
t = time.time()
total_ready = 0
total_unack = 0
rows = out.strip('\n').split('\n')[1:-1]
events = []
for row in rows:
name, ready, unack = row.split()
ready = int(ready)
unack = int(unack)
total_ready += ready
total_unack += unack
events.extend([
self.createEvent('ok', '%s unacknowledged messages: %s' % (
name, unack), unack, prefix='%s.unack' % name),
self.createEvent('ok', '%s ready messages: %s' % (
name, ready), ready, prefix='%s.ready' % name)
])
if name in self.ready:
last_ready = self.ready[name]
last_unack = self.unack[name]
rrate = (ready - last_ready)/float(t - self.last_t)
urate = (unack - last_unack)/float(t - self.last_t)
events.extend([
self.createEvent('ok', '%s unacknowledged rate: %0.2f' % (
name, urate), urate, prefix='%s.unack_rate' % name),
self.createEvent('ok', '%s ready rate: %0.2f' % (
name, rrate), rrate, prefix='%s.ready_rate' % name)
])
self.ready[name] = ready
self.unack[name] = unack
if self.last_t:
# Get total rates
rrate = (total_ready - self.last_ready)/float(t - self.last_t)
urate = (total_unack - self.last_unack)/float(t - self.last_t)
events.extend([
self.createEvent('ok',
'Total unacknowledged rate: %0.2f' % urate,
urate, prefix='total.unack_rate'),
self.createEvent('ok',
'Total ready rate: %0.2f' % rrate,
rrate, prefix='total.ready_rate'),
self.createEvent('ok',
'Total unacknowledged messages: %s' % total_unack,
total_unack, prefix='total.unack'),
self.createEvent('ok',
'Total ready messages: %s' % total_ready,
total_ready, prefix='total.ready')
])
self.last_ready = total_ready
self.last_unack = total_unack
self.last_t = t
defer.returnValue(events)
else:
log.msg('Error running rabbitmqctl: ' + repr(err))
def get(self):
out, err, code = yield fork('/bin/ps',
args=('-eo',
'pid,user,etime,rss,pcpu,comm,cmd'))
lines = out.strip('\n').split('\n')
cols = lines[0].split()
procs = {}
users = {}
vals = []
for l in lines[1:]:
parts = l.split(None, len(cols) - 1)
proc = {}
for i, e in enumerate(parts):
proc[cols[i]] = e.strip()
parts = None
elapsed = proc['ELAPSED']
if '-' in elapsed:
days = int(elapsed.split('-')[0])
hours, minutes, seconds = [
int(i) for i in elapsed.split('-')[1].split(':')
]
age = (days * 24 * 60 * 60) + (hours * 60 * 60) + (minutes *
60)
age += seconds
elif elapsed.count(':') == 2:
hours, minutes, seconds = [int(i) for i in elapsed.split(':')]
age = (hours * 60 * 60) + (minutes * 60) + seconds
else:
minutes, seconds = [int(i) for i in elapsed.split(':')]
age = (minutes * 60) + seconds
# Ignore kernel and tasks that just started, usually it's this ps
if (proc['CMD'][0] != '[') and (age > 0):
binary = proc['CMD'].split()[0].split('/')[-1].strip(
':').strip('-')
pid = proc['PID']
cmd = proc['CMD']
comm = proc['COMMAND']
user = proc['USER']
mem = int(proc['RSS']) / 1024.0
cpu = float(proc['%CPU'])
if user in users:
users[user]['cpu'] += cpu
users[user]['mem'] += mem
else:
users[user] = {'cpu': cpu, 'mem': mem}
if binary != comm:
key = "%s.%s" % (binary, comm)
else:
key = comm
if key in procs:
procs[key]['cpu'] += cpu
procs[key]['mem'] += mem
procs[key]['age'] += age
else:
procs[key] = {'cpu': cpu, 'mem': mem, 'age': age}
events = []
for k, v in users.items():
events.append(
self.createEvent('ok',
'User memory %s: %0.2fMB' % (k, v['mem']),
v['mem'],
prefix="user.%s.mem" % k))
events.append(
self.createEvent('ok',
'User CPU usage %s: %s%%' %
(k, int(v['cpu'] * 100)),
v['cpu'],
prefix="user.%s.cpu" % k))
for k, v in procs.items():
events.append(
self.createEvent('ok',
'Process age %s: %ss' % (k, v['age']),
v['age'],
prefix="proc.%s.age" % k))
events.append(
self.createEvent('ok',
'Process memory %s: %0.2fMB' % (k, v['mem']),
v['mem'],
prefix="proc.%s.mem" % k))
events.append(
self.createEvent('ok',
'Process CPU usage %s: %s%%' %
(k, int(v['cpu'] * 100)),
v['cpu'],
prefix="proc.%s.cpu" % k))
defer.returnValue(events)
def get(self):
vhost = self.config.get('vhost', '/')
mqctl = self.config.get('rabbitmqctl', '/usr/sbin/rabbitmqctl')
out, err, code = yield fork(mqctl,
args=('list_queues', '-p', vhost, 'name',
'messages_ready',
'messages_unacknowledged'))
if code == 0:
t = time.time()
total_ready = 0
total_unack = 0
rows = out.strip('\n').split('\n')
events = []
for row in rows:
if ("..." in row):
continue
name, ready, unack = row.split()
ready = int(ready)
unack = int(unack)
total_ready += ready
total_unack += unack
events.extend([
self.createEvent('ok',
'%s unacknowledged messages: %s' %
(name, unack),
unack,
prefix='%s.unack' % name),
self.createEvent('ok',
'%s ready messages: %s' % (name, ready),
ready,
prefix='%s.ready' % name)
])
if name in self.ready:
last_ready = self.ready[name]
last_unack = self.unack[name]
rrate = (ready - last_ready) / float(t - self.last_t)
urate = (unack - last_unack) / float(t - self.last_t)
events.extend([
self.createEvent('ok',
'%s unacknowledged rate: %0.2f' %
(name, urate),
urate,
prefix='%s.unack_rate' % name),
self.createEvent('ok',
'%s ready rate: %0.2f' %
(name, rrate),
rrate,
prefix='%s.ready_rate' % name)
])
self.ready[name] = ready
self.unack[name] = unack
if self.last_t:
# Get total rates
rrate = (total_ready - self.last_ready) / float(t -
self.last_t)
urate = (total_unack - self.last_unack) / float(t -
self.last_t)
events.extend([
self.createEvent('ok',
'Total unacknowledged rate: %0.2f' %
urate,
urate,
prefix='total.unack_rate'),
self.createEvent('ok',
'Total ready rate: %0.2f' % rrate,
rrate,
prefix='total.ready_rate'),
self.createEvent('ok',
'Total unacknowledged messages: %s' %
total_unack,
total_unack,
prefix='total.unack'),
self.createEvent('ok',
'Total ready messages: %s' % total_ready,
total_ready,
prefix='total.ready')
])
self.last_ready = total_ready
self.last_unack = total_unack
self.last_t = t
defer.returnValue(events)
else:
log.msg('Error running rabbitmqctl: ' + repr(err))
def _get_sensors(self):
out, err, code = yield fork('/usr/bin/sensors')
if code == 0:
defer.returnValue(out.strip('\n').split('\n'))
else:
defer.returnValue([])
def get(self):
out, err, code = yield fork('/bin/ps', args=(
'-eo','pid,user,etime,rss,pcpu,comm,cmd'))
lines = out.strip('\n').split('\n')
cols = lines[0].split()
procs = {}
users = {}
vals = []
for l in lines[1:]:
parts = l.split(None, len(cols) - 1)
proc = {}
for i, e in enumerate(parts):
proc[cols[i]] = e.strip()
parts = None
elapsed = proc['ELAPSED']
if '-' in elapsed:
days = int(elapsed.split('-')[0])
hours, minutes, seconds = [
int (i) for i in elapsed.split('-')[1].split(':')]
age = (days*24*60*60) + (hours*60*60) + (minutes*60)
age += seconds
elif elapsed.count(':')==2:
hours, minutes, seconds = [
int (i) for i in elapsed.split(':')]
age = (hours*60*60) + (minutes*60) + seconds
else:
minutes, seconds = [
int (i) for i in elapsed.split(':')]
age = (minutes*60) + seconds
# Ignore kernel and tasks that just started, usually it's this ps
if (proc['CMD'][0] != '[') and (age>0):
binary = proc['CMD'].split()[0].split('/')[-1].strip(':').strip('-')
pid = proc['PID']
cmd = proc['CMD']
comm = proc['COMMAND']
user = proc['USER']
mem = int(proc['RSS'])/1024.0
cpu = float(proc['%CPU'])
if user in users:
users[user]['cpu'] += cpu
users[user]['mem'] += mem
else:
users[user] = {
'cpu': cpu, 'mem': mem
}
if binary != comm:
key = "%s.%s" % (binary,comm)
else:
key = comm
if key in procs:
procs[key]['cpu'] += cpu
procs[key]['mem'] += mem
procs[key]['age'] += age
else:
procs[key] = {
'cpu': cpu, 'mem': mem, 'age': age
}
events = []
for k,v in users.items():
events.append(self.createEvent('ok', 'User memory %s: %0.2fMB' % (
k, v['mem']), v['mem'], prefix="user.%s.mem" % k))
events.append(self.createEvent('ok', 'User CPU usage %s: %s%%' % (
k, int(v['cpu']*100)), v['cpu'], prefix="user.%s.cpu" % k))
for k,v in procs.items():
events.append(self.createEvent('ok', 'Process age %s: %ss' % (
k, v['age']), v['age'], prefix="proc.%s.age" % k))
events.append(self.createEvent('ok', 'Process memory %s: %0.2fMB' % (
k, v['mem']), v['mem'], prefix="proc.%s.mem" % k))
events.append(
self.createEvent('ok', 'Process CPU usage %s: %s%%' % (
k, int(v['cpu']*100)), v['cpu'],
prefix="proc.%s.cpu" % k
)
)
defer.returnValue(events)
def test_utils_fork(self):
o, e, c = yield fork('echo', args=('hi',))
self.assertEquals(o, "hi\n")
self.assertEquals(c, 0)
def test_utils_fork(self):
o, e, c = yield fork('echo', args=('hi',))
self.assertEquals(o, "hi\n")
self.assertEquals(c, 0)
def fork(self, *a, **kw):
if self.use_ssh:
return self.ssh_client.fork(*a, **kw)
else:
return fork(*a, **kw)