Beispiel #1
0
    def get(self):
        host = self.config.get('destination', self.hostname)

        out, err, code = yield fork('/bin/ping',
                                    args=('-q', '-n', '-c', '5', '-i', '0.2',
                                          host),
                                    timeout=30.0)

        if code == 0:
            # Successful ping
            try:
                out = out.strip('\n').split('\n')[-2:]
                loss = int(out[0].split()[5].strip('%'))

                stat = out[1].split()[3].split('/')
                pmin, avg, pmax, mdev = [float(i) for i in stat]

                event = [
                    self.createEvent('ok',
                                     'Latency to %s' % host,
                                     avg,
                                     prefix="latency"),
                    self.createEvent('ok',
                                     '%s%% loss to %s' % (loss, host),
                                     loss,
                                     prefix="loss"),
                ]
            except Exception, e:
                print("Could not parse response %s" % repr(out))
                event = None
Beispiel #2
0
    def get(self):
        disks = self.config.get('disks')

        out, err, code = yield fork('/bin/df', args=('-lPx', 'tmpfs',))

        out = [i.split() for i in out.strip('\n').split('\n')[1:]]

        events = []

        for disk, size, used, free, util, mount in out:
            if disks and (disk not in disks):
                continue

            if disk != "udev":
                util = int(util.strip('%'))
                used = int(used)
                free = int(free)
                events.extend([
                    self.createEvent('ok', 'Disk usage %s%%' % (util),
                                     util, prefix=disk),
                    self.createEvent('ok', 'Disk usage %s kB' % (used),
                                     used, prefix="%s.bytes" % disk),
                    self.createEvent('ok', 'Disk free %s kB' % (free),
                                     free, prefix="%s.free" % disk)
                ])

        defer.returnValue(events)
Beispiel #3
0
    def get(self):
        out, err, code = yield fork('/bin/ps', args=('-e', ))

        count = len(out.strip('\n').split('\n')) - 1

        defer.returnValue(
            self.createEvent('ok', 'Process count %s' % (count), count))
Beispiel #4
0
    def get(self):
        host = self.config.get('destination', self.hostname)

        t0 = time.time()
        try:
            out, err, code = yield fork('/usr/bin/avprobe',
                args=('rtsp://%s/sample_100kbit.mp4' % host, ), timeout=30.0)
        except:
            code = 1
            err = None

        t_delta = (time.time() - t0) * 1000

        if code == 0:
            e = self.createEvent('ok', 'RTSP Request time to %s' % host, 
                    t_delta)
        else:
            if err:
                try:
                    error = err.strip('\n').split('\n')[-2]
                except:
                    error = err.replace('\n', '-')
            else:
                error = "Execution error"

            e = self.createEvent('critical', 
                    'Unable to stream %s:%s' % (host, error),
                    t_delta)

        defer.returnValue(e)
Beispiel #5
0
    def get(self):
        host = self.config.get('destination', self.hostname)

        t0 = time.time()
        try:
            out, err, code = yield fork('/usr/bin/avprobe',
                args=('rtsp://%s/sample_100kbit.mp4' % host, ), timeout=30.0)
        except:
            code = 1
            err = None

        t_delta = (time.time() - t0) * 1000

        if code == 0:
            e = self.createEvent('ok', 'RTSP Request time to %s' % host, 
                    t_delta)
        else:
            if err:
                try:
                    error = err.strip('\n').split('\n')[-2]
                except:
                    error = err.replace('\n', '-')
            else:
                error = "Execution error"

            e = self.createEvent('critical', 
                    'Unable to stream %s:%s' % (host, error),
                    t_delta)

        defer.returnValue(e)
Beispiel #6
0
    def _get_uc_stats(self):
        out, err, code = yield fork(self.uc, args=('stats', ))

        if code == 0:
            defer.returnValue(out.strip('\n').split('\n'))
        else:
            log.msg('Error running unbound-control: ' + repr(err))
            defer.returnValue([])
Beispiel #7
0
    def _get_smart(self, device):
        out, err, code = yield fork('/usr/sbin/smartctl',
            args=('-A', device))

        if code == 0:
            defer.returnValue(out.strip('\n').split('\n'))
        else:
            defer.returnValue([])
Beispiel #8
0
    def get(self):
        out, err, code = yield fork('/bin/ps', args=('-e',))

        count = len(out.strip('\n').split('\n')) - 1

        defer.returnValue(
            self.createEvent('ok', 'Process count %s' % (count), count)
        )
Beispiel #9
0
    def test_utils_fork_timeout(self):
        died = False
        try:
            o, e, c = yield fork('sleep', args=('2',), timeout=0.1)
        except:
            died = True

        self.assertTrue(died)
Beispiel #10
0
    def test_utils_fork_timeout(self):
        died = False
        try:
            o, e, c = yield fork('sleep', args=('2',), timeout=0.1)
        except error.ProcessTerminated:
            died = True

        self.assertTrue(died)
Beispiel #11
0
    def getVideo(self, url, out, time=10):
        try:
            if os.path.exists(out):
                os.unlink(out)

            out, err, code = yield utils.fork('/usr/bin/avconv',
                                              args=('-i', url, '-t', str(time),
                                                    '-c', 'copy', out),
                                              timeout=60.0)
        except Exception, e:
            out, err, code = "", str(e), 1
Beispiel #12
0
    def get(self):
        out, err, code = yield fork('/usr/bin/sudo',
                                    args=('ipsec', 'statusall'))

        connections = {}

        s = 0

        for l in out.strip('\n').split('\n'):
            if l == "Connections:":
                s = 1
                continue
            elif l == "Routed Connections:":
                s = 2
            elif "Security Associations" in l:
                s = 3
            elif l[0] == ' ' and ':' in l:
                if s == 1:
                    con, detail = l.strip().split(': ', 1)
                    detail = detail.strip()

                    if con not in connections:
                        connections[con] = {
                            'source': detail.split('...')[0],
                            'destination': detail.split('...')[1].split()[0],
                            'up': False
                        }
                elif s == 3:
                    con, detail = l.strip().split(': ', 1)
                    detail = detail.strip()
                    if '[' in con:
                        con = con.split('[')[0]
                    else:
                        con = con.split('{')[0]

                    if 'ESTABLISHED' in detail:
                        connections[con]['up'] = True

        events = []
        for k, v in connections.items():
            if v['up']:
                events.append(
                    self.createEvent('ok',
                                     'IPSec tunnel %s up' % k,
                                     1,
                                     prefix=k))
            else:
                events.append(
                    self.createEvent('critical',
                                     'IPSec tunnel %s down' % k,
                                     0,
                                     prefix=k))

        defer.returnValue(events)
Beispiel #13
0
    def _get_disks(self):
        out, err, code = yield fork('/usr/sbin/smartctl',
            args=('--scan',))

        if code != 0:
            defer.returnValue([])

        out = out.strip('\n').split('\n')
        devices = []
        for ln in out:
            if '/dev' in ln:
                devices.append(ln.split()[0])

        defer.returnValue(devices)
Beispiel #14
0
    def get(self):
        out, err, code = yield fork('/bin/df', args=('-lPx','tmpfs',))

        out = [i.split() for i in out.strip('\n').split('\n')[1:]]

        events = []

        for disk, size, used, free, util, mount in out:
            if disk != "udev":
                util = int(util.strip('%'))
                events.append(
                    self.createEvent('ok', 'Disk usage %s' % (util),
                        util, prefix=disk)
                )

        defer.returnValue(events)
Beispiel #15
0
    def get(self):
        out, err, code = yield fork("/usr/bin/sudo", args=("ipsec", "statusall"))

        connections = {}

        s = 0

        for l in out.strip("\n").split("\n"):
            if l == "Connections:":
                s = 1
                continue
            elif l == "Routed Connections:":
                s = 2
            elif "Security Associations" in l:
                s = 3
            elif l[0] == " " and ":" in l:
                if s == 1:
                    con, detail = l.strip().split(": ", 1)
                    detail = detail.strip()

                    if con not in connections:
                        connections[con] = {
                            "source": detail.split("...")[0],
                            "destination": detail.split("...")[1].split()[0],
                            "up": False,
                        }
                elif s == 3:
                    con, detail = l.strip().split(": ", 1)
                    detail = detail.strip()
                    if "[" in con:
                        con = con.split("[")[0]
                    else:
                        con = con.split("{")[0]

                    if "ESTABLISHED" in detail:
                        connections[con]["up"] = True

        events = []
        for k, v in connections.items():
            if v["up"]:
                events.append(self.createEvent("ok", "IPSec tunnel %s up" % k, 1, prefix=k))
            else:
                events.append(self.createEvent("critical", "IPSec tunnel %s down" % k, 0, prefix=k))

        defer.returnValue(events)
Beispiel #16
0
    def get(self):
        out, err, code = yield fork('/bin/df', args=(
            '-lPx',
            'tmpfs',
        ))

        out = [i.split() for i in out.strip('\n').split('\n')[1:]]

        events = []

        for disk, size, used, free, util, mount in out:
            if disk != "udev":
                util = int(util.strip('%'))
                events.append(
                    self.createEvent('ok',
                                     'Disk usage %s' % (util),
                                     util,
                                     prefix=disk))

        defer.returnValue(events)
Beispiel #17
0
    def get(self):

        out, err, code = yield fork(self.clipath,
            args=('-n', str(self.db), 'llen', self.queue,)
        )

        events = []
        if code == 0:
            val = int(out.strip('\n').split()[-1])

            defer.returnValue([
                self.createEvent('ok', '%s queue length' % self.queue, val),

                self.createEvent('ok', 'Queue rate', val, prefix='rate',
                    aggregation=Counter)
            ])

        else:
            err = 'Error running %s: %s' % (self.clipath, repr(err))
            log.msg(err)
            defer.returnValue(
                self.createEvent('critical', err, None)
            )
Beispiel #18
0
    def get(self):
        host = self.config.get('destination', self.hostname)

        out, err, code = yield fork('/bin/ping',
            args=('-q', '-n', '-c', '5', '-i', '0.2', host), timeout=30.0)

        if code == 0:
            # Successful ping
            try:
                out = out.strip('\n').split('\n')[-2:]
                loss = int(out[0].split()[5].strip('%'))

                stat = out[1].split()[3].split('/')
                pmin, avg, pmax, mdev = [float(i) for i in stat]

                event = [
                    self.createEvent('ok', 'Latency to %s' % host, avg,
                        prefix="latency"),
                    self.createEvent('ok', '%s%% loss to %s' % (loss,host), loss,
                        prefix="loss"),
                ]
            except Exception, e:
                print("Could not parse response %s" % repr(out))
                event = None
Beispiel #19
0
    def get(self):
        disks = self.config.get('disks')

        out, err, code = yield fork('/bin/df', args=(
            '-lPx',
            'tmpfs',
        ))

        out = [i.split() for i in out.strip('\n').split('\n')[1:]]

        events = []

        for disk, size, used, free, util, mount in out:
            if disks and (disk not in disks):
                continue

            if disk != "udev":
                util = int(util.strip('%'))
                used = int(used)
                free = int(free)
                events.extend([
                    self.createEvent('ok',
                                     'Disk usage %s%%' % (util),
                                     util,
                                     prefix=disk),
                    self.createEvent('ok',
                                     'Disk usage %s kB' % (used),
                                     used,
                                     prefix="%s.bytes" % disk),
                    self.createEvent('ok',
                                     'Disk free %s kB' % (free),
                                     free,
                                     prefix="%s.free" % disk)
                ])

        defer.returnValue(events)
Beispiel #20
0
    def get(self):
        vhost = self.config.get('vhost', '/')

        mqctl = self.config.get('rabbitmqctl', '/usr/sbin/rabbitmqctl')

        out, err, code = yield fork(mqctl, args=(
            'list_queues', '-p', vhost, 'name', 'messages_ready',
            'messages_unacknowledged'
        ))

        if code == 0:
            t = time.time()

            total_ready = 0
            total_unack = 0

            rows = out.strip('\n').split('\n')[1:-1]

            events = []

            for row in rows:
                name, ready, unack = row.split()
                ready = int(ready)
                unack = int(unack)

                total_ready += ready
                total_unack += unack

                events.extend([
                    self.createEvent('ok', '%s unacknowledged messages: %s' % (
                        name, unack), unack, prefix='%s.unack' % name),
                    self.createEvent('ok', '%s ready messages: %s' % (
                        name, ready), ready, prefix='%s.ready' % name)
                ])

                if name in self.ready:
                    last_ready = self.ready[name]
                    last_unack = self.unack[name]

                    rrate = (ready - last_ready)/float(t - self.last_t)
                    urate = (unack - last_unack)/float(t - self.last_t)

                    events.extend([
                        self.createEvent('ok', '%s unacknowledged rate: %0.2f' % (
                            name, urate), urate, prefix='%s.unack_rate' % name),
                        self.createEvent('ok', '%s ready rate: %0.2f' % (
                            name, rrate), rrate, prefix='%s.ready_rate' % name)
                    ])

                self.ready[name] = ready
                self.unack[name] = unack

            if self.last_t:
                # Get total rates
                rrate = (total_ready - self.last_ready)/float(t - self.last_t)
                urate = (total_unack - self.last_unack)/float(t - self.last_t)

                events.extend([
                    self.createEvent('ok', 
                        'Total unacknowledged rate: %0.2f' % urate,
                        urate, prefix='total.unack_rate'),
                    self.createEvent('ok', 
                        'Total ready rate: %0.2f' % rrate,
                        rrate, prefix='total.ready_rate'),
                    self.createEvent('ok', 
                        'Total unacknowledged messages: %s' % total_unack,
                        total_unack, prefix='total.unack'),
                    self.createEvent('ok', 
                        'Total ready messages: %s' % total_ready,
                        total_ready, prefix='total.ready')
                ])

            self.last_ready = total_ready
            self.last_unack = total_unack

            self.last_t = t

            defer.returnValue(events)
        else:
            log.msg('Error running rabbitmqctl: ' + repr(err))
Beispiel #21
0
    def get(self):
        out, err, code = yield fork('/bin/ps',
                                    args=('-eo',
                                          'pid,user,etime,rss,pcpu,comm,cmd'))

        lines = out.strip('\n').split('\n')

        cols = lines[0].split()

        procs = {}
        users = {}

        vals = []

        for l in lines[1:]:
            parts = l.split(None, len(cols) - 1)

            proc = {}
            for i, e in enumerate(parts):
                proc[cols[i]] = e.strip()

            parts = None

            elapsed = proc['ELAPSED']
            if '-' in elapsed:
                days = int(elapsed.split('-')[0])
                hours, minutes, seconds = [
                    int(i) for i in elapsed.split('-')[1].split(':')
                ]
                age = (days * 24 * 60 * 60) + (hours * 60 * 60) + (minutes *
                                                                   60)
                age += seconds

            elif elapsed.count(':') == 2:
                hours, minutes, seconds = [int(i) for i in elapsed.split(':')]
                age = (hours * 60 * 60) + (minutes * 60) + seconds

            else:
                minutes, seconds = [int(i) for i in elapsed.split(':')]
                age = (minutes * 60) + seconds

            # Ignore kernel and tasks that just started, usually it's this ps
            if (proc['CMD'][0] != '[') and (age > 0):
                binary = proc['CMD'].split()[0].split('/')[-1].strip(
                    ':').strip('-')
                pid = proc['PID']
                cmd = proc['CMD']
                comm = proc['COMMAND']
                user = proc['USER']

                mem = int(proc['RSS']) / 1024.0
                cpu = float(proc['%CPU'])

                if user in users:
                    users[user]['cpu'] += cpu
                    users[user]['mem'] += mem
                else:
                    users[user] = {'cpu': cpu, 'mem': mem}

                if binary != comm:
                    key = "%s.%s" % (binary, comm)
                else:
                    key = comm

                if key in procs:
                    procs[key]['cpu'] += cpu
                    procs[key]['mem'] += mem
                    procs[key]['age'] += age
                else:
                    procs[key] = {'cpu': cpu, 'mem': mem, 'age': age}

        events = []

        for k, v in users.items():
            events.append(
                self.createEvent('ok',
                                 'User memory %s: %0.2fMB' % (k, v['mem']),
                                 v['mem'],
                                 prefix="user.%s.mem" % k))
            events.append(
                self.createEvent('ok',
                                 'User CPU usage %s: %s%%' %
                                 (k, int(v['cpu'] * 100)),
                                 v['cpu'],
                                 prefix="user.%s.cpu" % k))

        for k, v in procs.items():
            events.append(
                self.createEvent('ok',
                                 'Process age %s: %ss' % (k, v['age']),
                                 v['age'],
                                 prefix="proc.%s.age" % k))
            events.append(
                self.createEvent('ok',
                                 'Process memory %s: %0.2fMB' % (k, v['mem']),
                                 v['mem'],
                                 prefix="proc.%s.mem" % k))
            events.append(
                self.createEvent('ok',
                                 'Process CPU usage %s: %s%%' %
                                 (k, int(v['cpu'] * 100)),
                                 v['cpu'],
                                 prefix="proc.%s.cpu" % k))

        defer.returnValue(events)
Beispiel #22
0
    def get(self):
        vhost = self.config.get('vhost', '/')

        mqctl = self.config.get('rabbitmqctl', '/usr/sbin/rabbitmqctl')

        out, err, code = yield fork(mqctl,
                                    args=('list_queues', '-p', vhost, 'name',
                                          'messages_ready',
                                          'messages_unacknowledged'))

        if code == 0:
            t = time.time()

            total_ready = 0
            total_unack = 0

            rows = out.strip('\n').split('\n')

            events = []

            for row in rows:
                if ("..." in row):
                    continue
                name, ready, unack = row.split()
                ready = int(ready)
                unack = int(unack)

                total_ready += ready
                total_unack += unack

                events.extend([
                    self.createEvent('ok',
                                     '%s unacknowledged messages: %s' %
                                     (name, unack),
                                     unack,
                                     prefix='%s.unack' % name),
                    self.createEvent('ok',
                                     '%s ready messages: %s' % (name, ready),
                                     ready,
                                     prefix='%s.ready' % name)
                ])

                if name in self.ready:
                    last_ready = self.ready[name]
                    last_unack = self.unack[name]

                    rrate = (ready - last_ready) / float(t - self.last_t)
                    urate = (unack - last_unack) / float(t - self.last_t)

                    events.extend([
                        self.createEvent('ok',
                                         '%s unacknowledged rate: %0.2f' %
                                         (name, urate),
                                         urate,
                                         prefix='%s.unack_rate' % name),
                        self.createEvent('ok',
                                         '%s ready rate: %0.2f' %
                                         (name, rrate),
                                         rrate,
                                         prefix='%s.ready_rate' % name)
                    ])

                self.ready[name] = ready
                self.unack[name] = unack

            if self.last_t:
                # Get total rates
                rrate = (total_ready - self.last_ready) / float(t -
                                                                self.last_t)
                urate = (total_unack - self.last_unack) / float(t -
                                                                self.last_t)

                events.extend([
                    self.createEvent('ok',
                                     'Total unacknowledged rate: %0.2f' %
                                     urate,
                                     urate,
                                     prefix='total.unack_rate'),
                    self.createEvent('ok',
                                     'Total ready rate: %0.2f' % rrate,
                                     rrate,
                                     prefix='total.ready_rate'),
                    self.createEvent('ok',
                                     'Total unacknowledged messages: %s' %
                                     total_unack,
                                     total_unack,
                                     prefix='total.unack'),
                    self.createEvent('ok',
                                     'Total ready messages: %s' % total_ready,
                                     total_ready,
                                     prefix='total.ready')
                ])

            self.last_ready = total_ready
            self.last_unack = total_unack

            self.last_t = t

            defer.returnValue(events)
        else:
            log.msg('Error running rabbitmqctl: ' + repr(err))
Beispiel #23
0
 def _get_sensors(self):
     out, err, code = yield fork('/usr/bin/sensors')
     if code == 0:
         defer.returnValue(out.strip('\n').split('\n'))
     else:
         defer.returnValue([])
Beispiel #24
0
    def get(self):
        out, err, code = yield fork('/bin/ps', args=(
            '-eo','pid,user,etime,rss,pcpu,comm,cmd'))

        lines = out.strip('\n').split('\n')

        cols = lines[0].split()

        procs = {}
        users = {}

        vals = []

        for l in lines[1:]:
            parts = l.split(None, len(cols) - 1)

            proc = {}
            for i, e in enumerate(parts):
                proc[cols[i]] = e.strip()

            parts = None

            elapsed = proc['ELAPSED']
            if '-' in elapsed:
                days = int(elapsed.split('-')[0])
                hours, minutes, seconds = [
                    int (i) for i in elapsed.split('-')[1].split(':')]
                age = (days*24*60*60) + (hours*60*60) + (minutes*60)
                age += seconds

            elif elapsed.count(':')==2:
                hours, minutes, seconds = [
                    int (i) for i in elapsed.split(':')]
                age = (hours*60*60) + (minutes*60) + seconds

            else:
                minutes, seconds = [
                    int (i) for i in elapsed.split(':')]
                age = (minutes*60) + seconds

            # Ignore kernel and tasks that just started, usually it's this ps
            if (proc['CMD'][0] != '[') and (age>0):
                binary = proc['CMD'].split()[0].split('/')[-1].strip(':').strip('-')
                pid = proc['PID']
                cmd = proc['CMD']
                comm = proc['COMMAND']
                user = proc['USER']

                mem = int(proc['RSS'])/1024.0
                cpu = float(proc['%CPU'])

                if user in users:
                    users[user]['cpu'] += cpu
                    users[user]['mem'] += mem
                else:
                    users[user] = {
                        'cpu': cpu, 'mem': mem
                    }

                if binary != comm:
                    key = "%s.%s" % (binary,comm)
                else:
                    key = comm

                if key in procs:
                    procs[key]['cpu'] += cpu
                    procs[key]['mem'] += mem
                    procs[key]['age'] += age
                else:
                    procs[key] = {
                        'cpu': cpu, 'mem': mem, 'age': age
                    }

        events = []

        for k,v in users.items():
            events.append(self.createEvent('ok', 'User memory %s: %0.2fMB' % (
                k, v['mem']), v['mem'], prefix="user.%s.mem" % k))
            events.append(self.createEvent('ok', 'User CPU usage %s: %s%%' % (
                k, int(v['cpu']*100)), v['cpu'], prefix="user.%s.cpu" % k))

        for k,v in procs.items():
            events.append(self.createEvent('ok', 'Process age %s: %ss' % (
                k, v['age']), v['age'], prefix="proc.%s.age" % k))
            events.append(self.createEvent('ok', 'Process memory %s: %0.2fMB' % (
                k, v['mem']), v['mem'], prefix="proc.%s.mem" % k))
            events.append(
                self.createEvent('ok', 'Process CPU usage %s: %s%%' % (
                    k, int(v['cpu']*100)), v['cpu'],
                    prefix="proc.%s.cpu" % k
                )
            )

        defer.returnValue(events)
Beispiel #25
0
    def test_utils_fork(self):
        o, e, c = yield fork('echo', args=('hi',))

        self.assertEquals(o, "hi\n")
        self.assertEquals(c, 0)
Beispiel #26
0
    def test_utils_fork(self):
        o, e, c = yield fork('echo', args=('hi',))

        self.assertEquals(o, "hi\n")
        self.assertEquals(c, 0)
Beispiel #27
0
 def fork(self, *a, **kw):
     if self.use_ssh:
         return self.ssh_client.fork(*a, **kw)
     else:
         return fork(*a, **kw)