예제 #1
0
    def setUp(self, mock_make_request):
        self.deviceobj = self.get_json('/json/device.json')

        self.client = ApiClient('aeou')
        self.client._make_request = mock_make_request
        self.client._make_request.return_value = self.deviceobj
        self.device = Device(api=self.client)
예제 #2
0
    def __init__(self, token, conf):
        self.token = token
        self.conf = conf
        self.historic_data = None
        client = ApiClient(self.token, timeout=(3.05, 18))
        self.device = Device(self.token)
        self.tag = Tag(self.token)
        self.metrics = Metrics(client)

        self._validation()
    def __init__(self, token, conf):
        self.token = token
        self.conf = conf
        self.historic_data = None
        client = ApiClient(self.token, timeout=(3.05, 18))
        self.device = Device(self.token)
        self.tag = Tag(self.token)
        self.metrics = Metrics(client)

        self._validation()
예제 #4
0
파일: devices.py 프로젝트: jonathan-s/sdbot
 def __init__(self):
     super(Wrapper, self).__init__()
     self.device = Device(self.token)
     self.metrics = Metrics(self.token)
예제 #5
0
파일: devices.py 프로젝트: jonathan-s/sdbot
class Wrapper(BaseWrapper):
    def __init__(self):
        super(Wrapper, self).__init__()
        self.device = Device(self.token)
        self.metrics = Metrics(self.token)

    def results_of(self, command, metrics, name):
        if command == 'help' or name == 'help':
            result = self.extra_help(command)
        elif command == 'find':
            result = self.find_device(name)
        elif command == 'value':
            result = self.get_value(name, metrics)
        elif command == 'available':
            result = self.get_available(name)
        elif command == 'list':
            result = self.list_devices(name)
        return result

    def extra_help(self, command):
        help_command = {
            'value': {
                'title': 'Latest Value for a Device',
                'mrkdwn_in': ['text'],
                'text': ('To get the latest value for a device, type ' +
                         '`sdbot devices metric.here for deviceName`. ' +
                         'The metrics need to be separated by dots.'),
                'color': COLOR
            },
            'find': {
                'title': 'Find a Device',
                'mrkdwn_in': ['text'],
                'text': ('To find a device type ' +
                         '`sdbot devices find deviceName`. I can also accept regex for the argument `deviceName`. ' +
                         'For example `sdbot devices find 2$`.'),
                'color': COLOR
            },
            'list': {
                'title': 'List Devices',
                'mrkdwn_in': ['text'],
                'text': ('To get a list of your devices type, ' +
                         '`sdbot devices list <no>`. In this case `<no>` ' +
                         'a number. If you leave it out I will ' +
                         'list the first 5 devices.'),
                'color': COLOR
            },
            'available': {
                'title': 'Available Metrics',
                'mrkdwn_in': ['text'],
                'text': ('To get all the available metrics for a device, type ' +
                         '`sdbot devices available deviceName`. This will ' +
                         'display a list of metrics you can use for the command `devices value` or `graph`'),
                'color': COLOR
            }
        }

        if command == 'value':
            helptext = [help_command['value']]
        elif command == 'available':
            helptext = [help_command['available']]
        elif command == 'find':
            helptext = [help_command['find']]
        elif command == 'list':
            helptext = [help_command['list']]
        elif command == 'help':
            helptext = [attachment for attachment in help_command.values()]
        return helptext

    def _format_devices(self, devices):
        formatted = [{
            'text': '*Device Name*: {}'.format(device['name']),
            'color': COLOR,
            'mrkdwn_in': ['text'],
            'fields': [{
                    'title': 'Group',
                    'value': device.get('group') if device.get('group') else 'Ungrouped',
                    'short': True
                },
                {
                    'title': 'Provider',
                    'value': device.get('provider') if device.get('provider') else 'No provider',
                    'short': True
                },
                {
                    'title': 'Id',
                    'value': device.get('_id'),
                    'short': True
                },
                {
                    'title': 'Online Status',
                    'value': self.online_status(device.get('lastPayloadAt', '')),
                    'short': True
                }
            ]
        } for device in devices]
        return formatted

    def list_devices(self, number):
        if number:
            try:
                number = number.strip()
                number = int(number)
            except ValueError:
                text = '{} is not a number, now is it. You see, it needs to be.'.format(number)
                return text
        devices = self.device.list()
        if number:
            devices_trunc = devices[:number]
        else:
            devices_trunc = devices[:5]

        return self._format_devices(devices_trunc)

    def find_device(self, name):
        devices = self.device.list()

        if not name:
            msg = 'Here are all the devices that I found'
            device_list = "\n".join([device['name'] for device in devices])
            result = msg + '\n```' + device_list + '```'
            return result

        devices = [device for device in devices if re.search(name, device['name'])]
        formatted_devices = self._format_devices(devices)

        if len(formatted_devices) == 0:
            formatted_devices = [{
                'text': 'Sorry, I couldn\'t find a device with that name :(',
                'color': COLOR
            }]

        return formatted_devices

    def get_value(self, name, metrics):
        devices = self.device.list()
        _id = self.find_id(name, [], devices)
        if not _id:
            return 'I couldn\'t find your device'

        if not metrics:
            return ('You have not included any metrics the right way to do it ' +
                    'is give metrics this way `sdbot devices value memory.memSwapFree for {}`'.format(name))
        metrics = metrics.split('.')
        _, filter = self.metric_filter(metrics)

        now = datetime.now()
        past30 = now - timedelta(minutes=35)

        metrics = self.metrics.get(_id, past30, now, filter)
        device, names = self.get_data(metrics)

        if not device.get('data'):
            return 'Could not find any data for these metrics'

        result = {
            'title': 'Device name: {}'.format(name),
            'text': ' > '.join(names),
            'color': COLOR,
            'fields': [
                {
                    'title': 'Latest Value',
                    'value': '{}{}'.format(device['data'][-1]['y'], self.extract_unit(device)),
                    'short': True
                }
            ]
        }
        return [result]

    def flatten(self, lst):
        for dct in lst:
            key = dct["key"]
            if "tree" not in dct:
                yield [key]  # base case
            else:
                for result in self.flatten(dct["tree"]):  # recursive case
                    yield [key] + result

    def get_available(self, name):
        devices = self.device.list()
        _id = self.find_id(name, [], devices)

        if not _id:
            return 'It looks like there is no device named `{}`'.format(name)
        now = datetime.now()
        past30 = now - timedelta(minutes=120)

        metrics = self.metrics.available(_id, past30, now)
        available = list(self.flatten(metrics))
        text = ''
        for a in available:
            text += '.'.join(a) + '\n'
        if text:
            text = 'Here are the metrics you can use\n' + '```' + text + '```'
        else:
            text = 'Your device seems to be offline, it doesn\'t contain any metrics in the last 2 hours'
        return text
예제 #6
0
 def __init__(self, msg, server):
     super(Wrapper, self).__init__(msg, server)
     self.device = Device(self.token)
     self.metrics = Metrics(self.token)
예제 #7
0
class Wrapper(BaseWrapper):
    def __init__(self, msg, server):
        super(Wrapper, self).__init__(msg, server)
        self.device = Device(self.token)
        self.metrics = Metrics(self.token)

    def results_of(self, command, metrics, name):
        if command == 'help' or name == 'help':
            result = self.extra_help(command)
        elif command == 'find':
            result = self.find_device(name)
        elif command == 'value':
            result = self.get_value(name, metrics)
        elif command == 'available':
            result = self.get_available(name)
        elif command == 'list':
            result = self.list_devices(name)
        return result

    def extra_help(self, command):
        help_command = {
            'value': {
                'title':
                'Latest Value for a Device',
                'mrkdwn_in': ['text'],
                'text': ('To get the latest value for a device, type ' +
                         '`sdbot devices metric.here for deviceName`. ' +
                         'The metrics need to be separated by dots.'),
                'color':
                COLOR
            },
            'find': {
                'title':
                'Find a Device',
                'mrkdwn_in': ['text'],
                'text':
                ('To find a device type ' +
                 '`sdbot devices find deviceName`. I can also accept regex for the argument `deviceName`. '
                 + 'For example `sdbot devices find 2$`.'),
                'color':
                COLOR
            },
            'list': {
                'title':
                'List Devices',
                'mrkdwn_in': ['text'],
                'text': ('To get a list of your devices type, ' +
                         '`sdbot devices list <no>`. In this case `<no>` ' +
                         'a number. If you leave it out I will ' +
                         'list the first 5 devices.'),
                'color':
                COLOR
            },
            'available': {
                'title':
                'Available Metrics',
                'mrkdwn_in': ['text'],
                'text':
                ('To get all the available metrics for a device, type ' +
                 '`sdbot devices available deviceName`. This will ' +
                 'display a list of metrics you can use for the command `devices value` or `graph`'
                 ),
                'color':
                COLOR
            }
        }

        if command == 'value':
            helptext = [help_command['value']]
        elif command == 'available':
            helptext = [help_command['available']]
        elif command == 'find':
            helptext = [help_command['find']]
        elif command == 'list':
            helptext = [help_command['list']]
        elif command == 'help':
            helptext = [attachment for attachment in help_command.values()]
        return helptext

    def _format_devices(self, devices):
        formatted = [{
            'text':
            '*Device Name*: {}'.format(device['name']),
            'color':
            COLOR,
            'mrkdwn_in': ['text'],
            'fields': [{
                'title':
                'Group',
                'value':
                device.get('group') if device.get('group') else 'Ungrouped',
                'short':
                True
            }, {
                'title':
                'Provider',
                'value':
                device.get('provider')
                if device.get('provider') else 'No provider',
                'short':
                True
            }, {
                'title': 'Id',
                'value': device.get('_id'),
                'short': True
            }, {
                'title':
                'Online Status',
                'value':
                self.online_status(device.get('lastPayloadAt', '')),
                'short':
                True
            }]
        } for device in devices]
        return formatted

    def list_devices(self, number):
        if number:
            try:
                number = number.strip()
                number = int(number)
            except ValueError:
                text = '{} is not a number, now is it. You see, it needs to be.'.format(
                    number)
                return text
        devices = self.device.list()
        if number:
            devices_trunc = devices[:number]
        else:
            devices_trunc = devices[:5]

        return self._format_devices(devices_trunc)

    def find_device(self, name):
        devices = self.device.list()

        if not name:
            msg = 'Here are all the devices that I found'
            device_list = "\n".join([device['name'] for device in devices])
            result = msg + '\n```' + device_list + '```'
            return result

        devices = [
            device for device in devices if re.search(name, device['name'])
        ]
        formatted_devices = self._format_devices(devices)

        if len(formatted_devices) == 0:
            formatted_devices = [{
                'text': 'Sorry, I couldn\'t find a device with that name :(',
                'color': COLOR
            }]

        return formatted_devices

    def get_value(self, name, metrics):
        devices = self.device.list()
        _id = self.find_id(name, [], devices)
        if not _id:
            return 'I couldn\'t find your device'

        if not metrics:
            return (
                'You have not included any metrics the right way to do it ' +
                'is give metrics this way `sdbot devices value memory.memSwapFree for {}`'
                .format(name))
        metrics = metrics.split('.')
        _, filter = self.metric_filter(metrics)

        now = datetime.now()
        past30 = now - timedelta(minutes=35)

        metrics = self.metrics.get(_id, past30, now, filter)
        device, names = self.get_data(metrics)

        if not device.get('data'):
            return 'Could not find any data for these metrics'

        result = {
            'title':
            'Device name: {}'.format(name),
            'text':
            ' > '.join(names),
            'color':
            COLOR,
            'fields': [{
                'title':
                'Latest Value',
                'value':
                '{}{}'.format(device['data'][-1]['y'],
                              self.extract_unit(device)),
                'short':
                True
            }]
        }
        return [result]

    def flatten(self, lst):
        for dct in lst:
            key = dct["key"]
            if "tree" not in dct:
                yield [key]  # base case
            else:
                for result in self.flatten(dct["tree"]):  # recursive case
                    yield [key] + result

    def get_available(self, name):
        devices = self.device.list()
        _id = self.find_id(name, [], devices)

        if not _id:
            return 'It looks like there is no device named `{}`'.format(name)
        now = datetime.now()
        past30 = now - timedelta(minutes=120)

        metrics = self.metrics.available(_id, past30, now)
        available = list(self.flatten(metrics))
        text = ''
        for a in available:
            text += '.'.join(a) + '\n'
        if text:
            text = 'Here are the metrics you can use\n' + '```' + text + '```'
        else:
            text = 'Your device seems to be offline, it doesn\'t contain any metrics in the last 2 hours'
        return text
예제 #8
0
 def __init__(self):
     super(Wrapper, self).__init__()
     self.device = Device(self.token)
     self.service = Service(self.token)
     self.alert = Alert(self.token)
예제 #9
0
class Wrapper(BaseWrapper):
    def __init__(self):
        super(Wrapper, self).__init__()
        self.device = Device(self.token)
        self.service = Service(self.token)
        self.alert = Alert(self.token)

    def results_of(self, command, typeof, name):
        if typeof == 'help' or command == 'help':
            result = self.extra_help(command)
        elif command == 'open alerts':
            result = self.list_alerts(command, typeof, name)
        elif command == 'devices':
            result = self.get_devices(typeof)
        elif command == 'services':
            result = self.get_services(typeof)
        return result

    def get_services(self, number):
        services = self.service.list()
        if number:
            services = services[:int(number)]
        else:
            services = services[:5]

        http = [s for s in services if s['checkType'] == 'http']
        tcp = [s for s in services if s['checkType'] == 'tcp']

        slack_http = [{
            'text': '*Service Name*: {}'.format(service['name']),
            'color': COLOR,
            'mrkdwn_in': ['text'],
            'fields': [{
                    'title': 'Group',
                    'value': service.get('group') if service.get('group') else 'Ungrouped',
                    'short': True
                },
                {
                    'title': 'Type of check',
                    'value': service.get('checkType'),
                    'short': True
                },
                {
                    'title': 'Url',
                    'value': service.get('checkUrl'),
                    'short': True
                },
                {
                    'title': 'Method',
                    'value': service.get('checkMethod'),
                    'short': True
                },
                {
                    'title': 'Slow threshold',
                    'value': str(service.get('slowThreshold')) + 'ms',
                    'short': True
                }
            ]
        } for service in http]

        slack_tcp = [{
            'text': '*Service Name*: {}'.format(service['name']),
            'color': COLOR,
            'mrkdwn_in': ['text'],
            'fields': [{
                    'title': 'Group',
                    'value': service.get('group') if service.get('group') else 'Ungrouped',
                    'short': True
                },
                {
                    'title': 'Type of check',
                    'value': service.get('checkType'),
                    'short': True
                },
                {
                    'title': 'Host',
                    'value': service.get('host'),
                    'short': True
                },
                {
                    'title': 'Port',
                    'value': service.get('port'),
                    'short': True
                }
            ]
        } for service in tcp]
        if not number:
            message = 'Here are 5 services, if you want to see more than that you can, just do `sdbot list services <number>`'
        else:
            message = 'Here are all your {} services'.format(len(slack_tcp + slack_http))
        return slack_tcp + slack_http, message

    def get_devices(self, number):
        devices = self.device.list()
        if number:
            devices = devices[:int(number)]
        else:
            devices = devices[:5]
        # list expression
        slack_formatting = [{
            'text': '*Device Name*: {}'.format(device['name']),
            'color': COLOR,
            'mrkdwn_in': ['text'],
            'fields': [{
                    'title': 'Group',
                    'value': device.get('group') if device.get('group') else 'Ungrouped',
                    'short': True
                },
                {
                    'title': 'Provider',
                    'value': device.get('provider') if device.get('provider') else 'No provider',
                    'short': True
                },
                {
                    'title': 'Id',
                    'value': device.get('_id'),
                    'short': True
                },
                {
                    'title': 'Online Status',
                    'value': self.online_status(device.get('lastPayloadAt', '')),
                    'short': True
                }
            ]
        } for device in devices]
        if not number:
            message = 'Here are 5 devices, if you want to see more than that you can, just do `sdbot list devices <number>`'
        else:
            message = 'Here are your devices'
        return slack_formatting, message

    def extra_help(self, command):
        help_command = {
            'open alerts': {
                'title': 'Open Alerts',
                'mrkdwn_in': ['text'],
                'text': ('The full command for `open alerts` is `open alerts' +
                         ' <type> <name>` where `type` is either a `device`' +
                         ', `service` or a `group`. `name` is the name of that ' +
                         'entity. Both `type` and `name` is optional. If ' +
                         'none is used I will give you 5 alerts by default, ' +
                         'if you want all alerts write `list open alerts all` ' +
                         'instead.'),
                'color': COLOR
            },
            'devices': {
                'title': 'Devices',
                'mrkdwn_in': ['text'],
                'text': ('The full command for `devices` is `devices <number>`' +
                         ', if a number is not specified I will give you 5 ' +
                         'devices by default'),
                'color': COLOR
            },
            'services': {
                'title': 'Services',
                'mrkdwn_in': ['text'],
                'text': ('The full command for listing services is ' +
                         '`sdbot list services <number>`, if a number is not ' +
                         'specified I will give you 5 services by default'),
                'color': COLOR
            }
        }

        if command == 'open alerts':
            helptext = [help_command['open alerts']]
        elif command == 'devices':
            helptext = [help_command['devices']]
        elif command == 'services':
            helptext = [help_command['services']]
        elif command == 'help':
            helptext = [attachment for attachment in help_command.values()]
        return helptext, ''

    def list_alerts(self, command, typeof, name):
        params = {
            'filter': {'fixed': False}
        }
        valid = ['service', 'device']
        if typeof and typeof in valid:
            params['filter']['config.subjectType'] = typeof
        elif typeof == 'group':
            params['filter']['subjectGroup'] = typeof
        else:
            text = 'Instead of `{}` you should have used `group`, `service`, `device` or `all`'.format(typeof), ''
            return text

        services = self.service.list()
        devices = self.device.list()

        if name:
            _id = name if not name else self.find_id(name, services, devices)
            params['filter']['config.subjectId'] = _id

        if typeof == 'group':
            _id = ''

        results = self.alert.triggered(params=params)
        alerts = sorted(results, key=lambda alert: alert['config']['lastTriggeredAt']['sec'], reverse=True)
        if not typeof or name:
            # When making a standard `list open alerts` we want to give a limited amount of alerts.
            stripped_alerts = alerts[:5]
        else:
            # we want to keep the entire list here
            stripped_alerts = alerts
        open_alerts = []
        for alert in stripped_alerts:
            field = alert['config']['fullName'].split(' > ')
            comparison = alert['config'].get('fullComparison') if alert['config'].get('fullComparison') else ''
            value = alert['config'].get('value') if alert['config'].get('value') else ''
            group = alert['config']['group'] if alert['config'].get('group') else 'Ungrouped'
            triggered_time = time.localtime(alert['config']['lastTriggeredAt']['sec'])

            _id = alert['config']['subjectId']
            name = self.find_name(_id, services, devices)
            attachment = {
                'title': '{}'.format(name),
                'text': '{} {} {}'.format(
                    field[1],
                    comparison,
                    value
                ),
                'color': COLOR,
                'fields': [
                    {
                        'title': 'Last triggered',
                        'value': time.strftime('%Y-%m-%d, %H:%M:%S', triggered_time)
                    },
                    {
                        'title': 'Group',
                        'value': group,
                        'short': True
                    }
                ]
            }
            open_alerts.append(attachment)
        if open_alerts:
            if len(alerts) > len(stripped_alerts):
                message = ('You have {} open alerts but I\'m only showing the last {},'.format(len(alerts), len(stripped_alerts)) +
                           ' do `list open alerts all` to see all of them')
            else:
                message = 'Chop chop, you\'d better sort out these open alerts soon'
            return open_alerts, message
        else:
            return 'I could not find any open alerts for you.', ''
예제 #10
0
class DeviceTest(BaseTest):

    @patch.object(ApiClient, '_make_request')
    def setUp(self, mock_make_request):
        self.deviceobj = self.get_json('/json/device.json')

        self.client = ApiClient('aeou')
        self.client._make_request = mock_make_request
        self.client._make_request.return_value = self.deviceobj
        self.device = Device(api=self.client)

    def test_device_create(self):
        data = {'data': 'device'}
        self.device.create(data)
        self.client._make_request.assert_called_with(
            data=data,
            method='POST',
            url=Device.PATHS['create'],
            params=None
        )

    def test_device_delete(self):
        self.device.delete(1)
        self.client._make_request.assert_called_with(
            data=None,
            method='DELETE',
            url=Device.PATHS['delete'].format(1),
            params=None
        )

    def test_device_list(self):
        self.client._make_request.return_value = [self.deviceobj]
        self.device.list()
        self.client._make_request.assert_called_with(
            data=None,
            method='GET',
            url=Device.PATHS['list'],
            params=None
        )

    def test_device_search(self):
        self.client._make_request.return_value = [self.deviceobj]
        filter_data = {'name': 'test', 'type': 'device'}
        self.device.search(filtering=filter_data)
        self.client._make_request.assert_called_with(
            data=None,
            method='GET',
            url=Device.PATHS['search'],
            params={'filter': filter_data}
        )

    def test_device_update(self):
        data = {'name': 'test', 'type': 'device'}
        self.device.update(_id=1, data=data)
        self.client._make_request.assert_called_with(
            data=data,
            method='PUT',
            url=Device.PATHS['update'].format(1),
            params=None
        )

    def test_device_view_by_agent(self):
        self.device.view_by_agent(1)
        self.client._make_request.assert_called_with(
            data=None,
            method='GET',
            url=Device.PATHS['view_by_agent'].format(1),
            params=None
        )

    def test_device_view(self):
        self.device.view(1)
        self.client._make_request.assert_called_with(
            data=None,
            method='GET',
            url=Device.PATHS['view'].format(1),
            params=None
        )
예제 #11
0
파일: graph.py 프로젝트: ccasado/sdbot
 def __init__(self, msg, server):
     super(Wrapper, self).__init__()
     self.metrics = Metrics(self.token)
     self.device = Device(self.token)
     self.server = server
     self.msg = msg
예제 #12
0
파일: graph.py 프로젝트: ccasado/sdbot
class Wrapper(BaseWrapper):
    def __init__(self, msg, server):
        super(Wrapper, self).__init__()
        self.metrics = Metrics(self.token)
        self.device = Device(self.token)
        self.server = server
        self.msg = msg

    def results_of(self, metrics, name, period):
        if name == 'help':
            mod = importlib.import_module('limbo.plugins.graph')
            result = [json.loads(mod.__doc__)]
        else:
            result = self.get_metrics(metrics, name, period)
        return result

    def create_graph(self, device, difference):

        # 800 and 355 pixels.
        ticks = 5
        width = 8
        height = 3.55

        dpi = 100
        bgcolor = '#f3f6f6'

        font = {
            'size': 16,
            'family': 'Arial'
        }
        plt.rc('font', **font)

        diff_sec = int(difference.total_seconds())

        if diff_sec < 86400:
            x_no_ticks = 10
            fmt = '%H:%M'
        elif (3600*24) < diff_sec and diff_sec < (3600*24*4):
            x_no_ticks = 5
            fmt = '%d %b, %H:%M'
        elif (3600*24*4) < diff_sec:
            x_no_ticks = 6
            fmt = '%d %b %Y'

        # size of figure and setting background color
        fig = plt.gcf()
        fig.set_size_inches(width, height)
        fig.set_facecolor(bgcolor)

        # axis color, no ticks and bottom line in grey color.
        ax = plt.axes(axisbg=bgcolor, frameon=True)
        ax.xaxis.set_ticks_position('none')
        ax.spines['bottom'].set_color('#aabcc2')
        ax.yaxis.set_ticks_position('none')

        # removing all but bottom spines
        for key, sp in ax.spines.items():
            if key != 'bottom':
                sp.set_visible(False)

        # setting amounts of ticks on y axis
        yloc = plt.MaxNLocator(ticks)
        ax.yaxis.set_major_locator(yloc)

        # Deciding how many ticks we want on the graph
        locator = AutoDateLocator(minticks=(x_no_ticks - 2), maxticks=x_no_ticks)
        formatter = DateFormatter(fmt)

        ax.xaxis.set_major_locator(locator)
        ax.xaxis.set_major_formatter(formatter)

        # turns off small ticks
        plt.tick_params(axis='x',
                        which='both',
                        bottom='on',
                        top='off',
                        pad=10)
        # Can't seem to set label color differently, changing tick_params color changes labels.
        ax.xaxis.label.set_color('#FFFFFF')

        # setting dates in x-axis automatically triggers use of AutoDateLocator
        x = [datetime.fromtimestamp(point['x']) for point in device['data']]
        y = [point['y'] for point in device['data']]
        plt.plot(x, y, color='#53b4d4', linewidth=2)

        # pick values for y-axis
        y_ticks_values = np.array([point['y'] for point in device['data']])
        y_ticks = np.linspace(y_ticks_values.min(), y_ticks_values.max(), ticks)
        y_ticks = np.round(y_ticks, decimals=2)
        plt.yticks(y_ticks, [str(val) + self.extract_unit(device) for val in y_ticks])
        # plt.ylim(ymin=0.1)  # Only show values of a certain threshold.

        plt.tight_layout()
        buf = io.BytesIO()
        plt.savefig(buf,
                    format='png',
                    facecolor=fig.get_facecolor(),
                    dpi=dpi)
        buf.seek(0)
        filename = 'graph-{}.png'.format(int(time.time()))
        with io.open(filename, 'wb') as f:
            f.write(buf.read())
        plt.close()
        return filename

    def get_metrics(self, metrics, name, period):
        devices = self.device.list()
        _id = self.find_id(name, [], devices)
        if not _id:
            return 'I couldn\'t find your device.'

        metrics_names = metrics.split('.')
        _, filter = self.metric_filter(metrics_names)

        cal = parsedatetime.Calendar()
        if not period:
            period = '2 hours ago'
        past, _ = cal.parseDT(datetimeString=period, tzinfo=self.timezone)
        now = datetime.now(self.timezone)

        if past > now:
            return 'Hey, I can\'t predict your data into the future, your date has to be in the past and now your date is {}'.format(past)

        metrics_data = self.metrics.get(_id, past, now, filter)
        device, names = self.get_data(metrics_data)
        if not device.get('data'):
            text = ('It might be that your device is offline or has no metrics for `{}`.'.format(metrics) +
                    'You can see what metrics are available by using `sdbot devices available {}`'.format(name))
            return text

        # creates file
        slack = Slacker(os.environ.get('SLACK_TOKEN'))
        slack.chat.post_message(
            self.msg['channel'],
            'Preparing the graphs for you this very moment',
            as_user=self.server.slack.server.username
        )

        filename = self.create_graph(device, now - past)

        attachment = [
            {
                'text': ('I brought you a graph for {} for the device `{}`'.format(' '.join(names), name) +
                         '\n A graph is coming up in just a sec.'),
                'mrkdwn_in': ['text'],
                'color': COLOR
            }
        ]

        slack.chat.post_message(
            self.msg['channel'],
            '',
            attachments=attachment,
            as_user=self.server.slack.server.username
        )

        # uploads and sends the graph to channel
        file_response = slack.files.upload(
            filename,
            filename='{}.png'.format(name),
            channels=self.msg['channel']
        )
        # Deletes the graph file after upload so it doesn't litter the container
        os.remove(filename)

        return None # We're sending information in function itself this time
예제 #13
0
class Wrapper(BaseWrapper):
    def __init__(self, msg, server):
        super(Wrapper, self).__init__(msg, server)
        self.alert = Alert(self.token)
        self.device = Device(self.token)
        self.service = Service(self.token)

    def results_of(self, command, typeof, name):
        if typeof == 'help' or command == 'help':
            result = self.extra_help(command)
        elif command == 'list':
            result = self.list_alerts(command, typeof, name)
        return result

    def extra_help(self, command):
        help_command = {
            'list': {
                'title':
                'List Open Alerts',
                'mrkdwn_in': ['text'],
                'text':
                ('The full command for alerts is `sdbot alerts list' +
                 ' <type> <name>`, where `type` is either the word `device`' +
                 ', `service` or `group`. The argument `name` corresponds to the '
                 +
                 'name of that entity. Both `type` and `name` are optional. If '
                 + 'none is used I will give you 5 alerts by default. ' +
                 'If you want all alerts write, type `sdbot alerts list all`.'
                 ),
                'color':
                COLOR
            }
        }

        if command == 'list':
            helptext = [help_command['list']]
        elif command == 'help':
            helptext = [attachment for attachment in help_command.values()]
        return helptext, ''

    def _is_mongoId(self, _id):
        if len(_id) == 24:
            return True
        return False

    def list_alerts(self, command, typeof, name):
        params = {'filter': {'fixed': False}}
        valid = ['service', 'device']
        if typeof and typeof in valid:
            params['filter']['config.subjectType'] = typeof
        elif typeof == 'group':
            params['filter']['subjectGroup'] = name
        elif typeof in ['all', '']:
            pass  # don't need to do anything
        else:
            text = 'Instead of `{}` you should have used `group`, `service`, `device` or `all`'.format(
                typeof), ''
            return text

        services = self.service.list()
        devices = self.device.list()

        if name and typeof != 'group':
            _id = name if not name else self.find_id(name, services, devices)
            params['filter']['config.subjectId'] = _id

        results = self.alert.triggered(params=params)
        alerts = sorted(
            results,
            key=lambda alert: alert['config']['lastTriggeredAt']['sec'],
            reverse=True)
        if not (typeof or name):
            # When making a standard `alerts list` we want to give a limited amount of alerts.
            stripped_alerts = alerts[:5]
        else:
            # we want to keep the entire list here
            stripped_alerts = alerts
        open_alerts = []

        for alert in stripped_alerts:
            field = alert['config']['fullName'].split(' > ')
            comparison = alert['config'].get('fullComparison', '')
            value = '{}{}'.format(alert['config'].get('value', ''),
                                  alert['config'].get('units', ''))
            group = alert['config'].get('group', 'Ungrouped')
            triggered_time = time.localtime(
                alert['config']['lastTriggeredAt']['sec'])

            _id = alert['config']['subjectId']
            if self._is_mongoId(_id):
                name = self.find_name(_id, services, devices)
                name = '{}: {}'.format(alert['config']['subjectType'].title(),
                                       name)
            else:
                name = 'Group: {}'.format(_id)

            attachment = {
                'title':
                '{}'.format(name),
                'text':
                '{} {} {}'.format(field[1], comparison, value),
                'color':
                COLOR,
                'fields': [
                    {
                        'title':
                        'Last triggered',
                        'value':
                        time.strftime('%Y-%m-%d, %H:%M:%S', triggered_time)
                    },
                    # waiting on backend bugfix sd-2190
                    # {
                    #     'title': 'Group',
                    #     'value': group,
                    #     'short': True
                    # }
                ]
            }
            open_alerts.append(attachment)
        if open_alerts:
            if len(alerts) > len(stripped_alerts):
                message = (
                    'You have {} open alerts but I\'m only showing the last {},'
                    .format(len(alerts), len(stripped_alerts)) +
                    ' do `list open alerts all` to see all of them')
            else:
                message = 'Chop chop, you\'d better sort out these open alerts soon'
            return open_alerts, message
        else:
            return 'I could not find any open alerts for you.', ''
예제 #14
0
class Wrapper(BaseWrapper):
    def __init__(self, msg, server):
        super(Wrapper, self).__init__(msg, server)
        self.alert = Alert(self.token)
        self.device = Device(self.token)
        self.service = Service(self.token)

    def results_of(self, command, typeof, name):
        if typeof == 'help' or command == 'help':
            result = self.extra_help(command)
        elif command == 'list':
            result = self.list_alerts(command, typeof, name)
        return result

    def extra_help(self, command):
        help_command = {
            'list': {
                'title': 'List Open Alerts',
                'mrkdwn_in': ['text'],
                'text': ('The full command for alerts is `sdbot alerts list' +
                         ' <type> <name>`, where `type` is either the word `device`' +
                         ', `service` or `group`. The argument `name` corresponds to the ' +
                         'name of that entity. Both `type` and `name` are optional. If ' +
                         'none is used I will give you 5 alerts by default. ' +
                         'If you want all alerts write, type `sdbot alerts list all`.'),
                'color': COLOR
            }
        }

        if command == 'list':
            helptext = [help_command['list']]
        elif command == 'help':
            helptext = [attachment for attachment in help_command.values()]
        return helptext, ''

    def _is_mongoId(self, _id):
        if len(_id) == 24:
            return True
        return False

    def list_alerts(self, command, typeof, name):
        params = {
            'filter': {'fixed': False}
        }
        valid = ['service', 'device']
        if typeof and typeof in valid:
            params['filter']['config.subjectType'] = typeof
        elif typeof == 'group':
            params['filter']['subjectGroup'] = name
        elif typeof in ['all', '']:
            pass  # don't need to do anything
        else:
            text = 'Instead of `{}` you should have used `group`, `service`, `device` or `all`'.format(typeof), ''
            return text

        services = self.service.list()
        devices = self.device.list()

        if name and typeof != 'group':
            _id = name if not name else self.find_id(name, services, devices)
            params['filter']['config.subjectId'] = _id

        results = self.alert.triggered(params=params)
        alerts = sorted(results, key=lambda alert: alert['config']['lastTriggeredAt']['sec'], reverse=True)
        if not (typeof or name):
            # When making a standard `alerts list` we want to give a limited amount of alerts.
            stripped_alerts = alerts[:5]
        else:
            # we want to keep the entire list here
            stripped_alerts = alerts
        open_alerts = []

        for alert in stripped_alerts:
            field = alert['config']['fullName'].split(' > ')
            comparison = alert['config'].get('fullComparison', '')
            value = '{}{}'.format(alert['config'].get('value', ''), alert['config'].get('units', ''))
            group = alert['config'].get('group', 'Ungrouped')
            triggered_time = time.localtime(alert['config']['lastTriggeredAt']['sec'])

            _id = alert['config']['subjectId']
            if self._is_mongoId(_id):
                name = self.find_name(_id, services, devices)
                name = '{}: {}'.format(alert['config']['subjectType'].title(), name)
            else:
                name = 'Group: {}'.format(_id)

            attachment = {
                'title': '{}'.format(name),
                'text': '{} {} {}'.format(
                    field[1],
                    comparison,
                    value
                ),
                'color': COLOR,
                'fields': [
                    {
                        'title': 'Last triggered',
                        'value': time.strftime('%Y-%m-%d, %H:%M:%S', triggered_time)
                    },
                    # waiting on backend bugfix sd-2190
                    # {
                    #     'title': 'Group',
                    #     'value': group,
                    #     'short': True
                    # }
                ]
            }
            open_alerts.append(attachment)
        if open_alerts:
            if len(alerts) > len(stripped_alerts):
                message = ('You have {} open alerts but I\'m only showing the last {},'.format(len(alerts), len(stripped_alerts)) +
                           ' do `list open alerts all` to see all of them')
            else:
                message = 'Chop chop, you\'d better sort out these open alerts soon'
            return open_alerts, message
        else:
            return 'I could not find any open alerts for you.', ''
예제 #15
0
 def __init__(self):
     super(Wrapper, self).__init__()
     self.device = Device(self.token)
     self.metrics = Metrics(self.token)
class DataWrapper(object):

    def __init__(self, token, conf):
        self.token = token
        self.conf = conf
        self.historic_data = None
        client = ApiClient(self.token, timeout=(3.05, 18))
        self.device = Device(self.token)
        self.tag = Tag(self.token)
        self.metrics = Metrics(client)

        self._validation()

    def _validation(self):
        """Check the following
        - metric for every piece in infrastructure
        - That the configuration contains everything necessary.
        - check that there are no none values in any lists.
        """

    def _get_devices(self, infra_conf):
        """Takes the configuration part for each infrastructure"""
        raw_devices = self.device.list()

        if infra_conf.get('tag'):
            tags = self.tag.list()
            tag_id =[tag['_id'] for tag in tags if tag['name'] == infra_conf['tag']]
            if not tag_id:
                available_tags = '\n'.join(list(set([tag['name'] for tag in tags])))
                message = 'There is no tag with the name "{}". Try one of these: \n{}'.format(infra_conf['tag'], available_tags)
                raise Exception(message)
            else:
                tag_id = tag_id[0]
            devices = [device for device in raw_devices if tag_id in device.get('tags', [])]
            if not devices:
                available_tags = '\n'.join(list(set([tag['name'] for tag in tags])))
                raise Exception('There is no device with this tag name. Try one of these: \n{}'.format(available_tags))
        elif infra_conf.get('group'):
            group = infra_conf.get('group')
            devices = [device for device in raw_devices if group == device.get('group')]
            if not devices:
                groups = set([device['group'] for device in raw_devices
                             if not device['group'] is None])
                groups = '\n'.join(list(groups))
                raise Exception('There is no device with this group name. The following groups are available:\n {}'.format(groups))
        else:
            raise Exception('You have to provide either group or tag for each part of your infrastructure.')
        return devices

    def _merge_loadbalanced_data(self, data_points, points):
        max_length = len(data_points)
        for i, point in enumerate(points):
            if i < max_length:
                data_points[i] = data_points[i] + point
        return data_points

    def _get_metrics(self, metric, devices):
        """For all devices associated with the group or device"""
        metric = metric.split('.')
        metric_filter = self.metric_filter(metric)
        end = dt.datetime.now()
        start = end - timedelta(hours=self.conf['general'].get('timeframe', 24))
        data_entries = []
        for device in devices:
            data = self.metrics.get(device['_id'], start, end, metric_filter)
            data = self._data_node(data)
            time.sleep(0.3)
            if data['data']:
                data_entries.append(data)

        if not data_entries:
            metric = '.'.join(metric)
            # Append zero data to avoid zerodivison error
            data_entries.append({'data': [{'x': 0, 'y': 0}]})
            logging.warning('No server in this group has any data on {}'.format(metric))
        return data_entries

    def _data_node(self, data, names=None):
        """Inputs the data from the metrics endpoints and returns
        the node that has contains the data + names of the metrics."""

        if not names:
            names = []
        for d in data:
            if d.get('data') or d.get('data') == []:
                names.append(d.get('name'))
                d['full_name'] = names
                return d
            else:
                names.append(d.get('name'))
                return self._data_node(d.get('tree'), names)

    def _get_data_points(self, cumulative, data_entries, multiplier):
        """Extract the singular points into a list and return a list of those points"""
        data_points = []
        for data in data_entries:
            points = [point['y'] * multiplier for point in data['data']]
            if cumulative and len(data_points) > 0:
                self._merge_loadbalanced_data(data_points, points)
            else:
                data_points.extend(points)
        return data_points

    def _round(self, number):
        rounding = self.conf['general'].get('round', 2)
        if rounding > 0:
            return round(number, rounding)
        else:
            return int(round(number, 0))

    def calc_average(self, data_points):
        return self._round(sum(data_points) / len(data_points))

    def calc_max(self, data_points):
        return self._round(max(data_points))

    def calc_min(self, data_points):
        return self._round(min(data_points))

    def calc_median(self, data_points):
        data_points.sort()
        start = len(data_points) // 2.0
        if len(data_points) % 2 > 0:
            result = (data_points[start] + data_points[start + 1]) / 2.0
        else:
            result = self._round(data_points[start] // 2.0)
        return result

    def calc_sum(self, data_points):
        return self._round(sum(data_points))

    def gather_data(self):
        """The main function that gathers all data and returns an updated
        configuration file that the index template can read"""

        infrastructure = self.conf['infrastructure']
        for infra_conf in infrastructure:
            logging.info('Gather data from {}...'.format(infra_conf['title']))
            devices = self._get_devices(infra_conf)
            for metric_conf in infra_conf['metrics']:
                metric = metric_conf.get('metrickey')
                cumulative = metric_conf.get('cumulative', True)
                # giving the option to show case static information in boxes
                if metric:
                    multiplier = metric_conf.get('multiplier', 1)
                    data_entries = self._get_metrics(metric, devices)
                    for method in metric_conf['calculation']:
                        data_points = self._get_data_points(cumulative, data_entries, multiplier)
                        result = getattr(self, 'calc_{}'.format(method))(data_points)
                        metric_conf['{}_stat'.format(method)] = result
        return self.conf

    def metric_filter(self, metrics, filter=None):
        """from a list of metrics ie ['cpuStats', 'CPUs', 'usr'] it constructs
        a dictionary that can be sent to the metrics endpoint for consumption"""

        metrics = list(metrics)
        if not filter:
            filter = {}
            filter[metrics.pop()] = 'all'
            return self.metric_filter(metrics, filter)
        else:
            try:
                metric = metrics.pop()
                dic = {metric: filter}
                return self.metric_filter(metrics, dic)
            except IndexError:
                return filter

    def available(self):
        """Assumes that all metrics are the same for a group or a tag"""
        infrastructure = self.conf['infrastructure']
        md = '# Available metrics for all your groups\n\n'

        for infra_conf in infrastructure:
            if infra_conf.get('group'):
                category = infra_conf['group']
            elif infra_conf.get('tag'):
                category = infra_conf['tags']
            else:
                raise Exception('You need to provide either a group or tag')
            logging.info('Gathering metrics from {}...'.format(category))
            devices = self._get_devices(infra_conf)
            device = devices[0]
            end = dt.datetime.now()
            start = end - timedelta(hours=2)
            available = self.metrics.available(device['_id'], start, end)
            metrics = self.flatten(available)
            try:
                md += '## {}\n'.format(infra_conf['title'])
            except KeyError:
                raise KeyError('Each section need a title, go on fill one in and try again.')
            for metric in metrics:
                title = ' '.join([tup[0] for tup in metric])
                metric = '.'.join([tup[1] for tup in metric])
                entry = '##### {}\nmetrickey: {}\n\n'.format(title, metric)
                md += entry
        with codecs.open('available.md', 'w') as f:
            f.write(md)

    def flatten(self, lst):
        """Get all the keys when calling available"""
        for dct in lst:
            key = dct['key']
            name = dct['name']
            if 'tree' not in dct:
                yield [(name, key)]  # base case
            else:
                for result in self.flatten(dct["tree"]):  # recursive case
                    yield [(name, key)] + result
예제 #17
0
class Wrapper(BaseWrapper):
    def __init__(self, msg, server):
        super(Wrapper, self).__init__()
        self.metrics = Metrics(self.token)
        self.device = Device(self.token)
        self.server = server
        self.msg = msg

    def results_of(self, metrics, name, period):
        if name == 'help':
            mod = importlib.import_module('limbo.plugins.graph')
            result = [json.loads(mod.__doc__)]
        else:
            result = self.get_metrics(metrics, name, period)
        return result

    def create_graph(self, device, difference):

        # 800 and 355 pixels.
        ticks = 5
        width = 8
        height = 3.55

        dpi = 100
        bgcolor = '#f3f6f6'

        font = {'size': 16, 'family': 'Arial'}
        plt.rc('font', **font)

        diff_sec = int(difference.total_seconds())

        if diff_sec < 86400:
            x_no_ticks = 10
            fmt = '%H:%M'
        elif (3600 * 24) < diff_sec and diff_sec < (3600 * 24 * 4):
            x_no_ticks = 5
            fmt = '%d %b, %H:%M'
        elif (3600 * 24 * 4) < diff_sec:
            x_no_ticks = 6
            fmt = '%d %b %Y'

        # size of figure and setting background color
        fig = plt.gcf()
        fig.set_size_inches(width, height)
        fig.set_facecolor(bgcolor)

        # axis color, no ticks and bottom line in grey color.
        ax = plt.axes(axisbg=bgcolor, frameon=True)
        ax.xaxis.set_ticks_position('none')
        ax.spines['bottom'].set_color('#aabcc2')
        ax.yaxis.set_ticks_position('none')

        # removing all but bottom spines
        for key, sp in ax.spines.items():
            if key != 'bottom':
                sp.set_visible(False)

        # setting amounts of ticks on y axis
        yloc = plt.MaxNLocator(ticks)
        ax.yaxis.set_major_locator(yloc)

        # Deciding how many ticks we want on the graph
        locator = AutoDateLocator(minticks=(x_no_ticks - 2),
                                  maxticks=x_no_ticks)
        formatter = DateFormatter(fmt)

        ax.xaxis.set_major_locator(locator)
        ax.xaxis.set_major_formatter(formatter)

        # turns off small ticks
        plt.tick_params(axis='x', which='both', bottom='on', top='off', pad=10)
        # Can't seem to set label color differently, changing tick_params color changes labels.
        ax.xaxis.label.set_color('#FFFFFF')

        # setting dates in x-axis automatically triggers use of AutoDateLocator
        x = [datetime.fromtimestamp(point['x']) for point in device['data']]
        y = [point['y'] for point in device['data']]
        plt.plot(x, y, color='#53b4d4', linewidth=2)

        # pick values for y-axis
        y_ticks_values = np.array([point['y'] for point in device['data']])
        y_ticks = np.linspace(y_ticks_values.min(), y_ticks_values.max(),
                              ticks)
        y_ticks = np.round(y_ticks, decimals=2)
        plt.yticks(y_ticks,
                   [str(val) + self.extract_unit(device) for val in y_ticks])
        # plt.ylim(ymin=0.1)  # Only show values of a certain threshold.

        plt.tight_layout()
        buf = io.BytesIO()
        plt.savefig(buf, format='png', facecolor=fig.get_facecolor(), dpi=dpi)
        buf.seek(0)
        filename = 'graph-{}.png'.format(int(time.time()))
        with io.open(filename, 'wb') as f:
            f.write(buf.read())
        plt.close()
        return filename

    def get_metrics(self, metrics, name, period):
        devices = self.device.list()
        _id = self.find_id(name, [], devices)
        if not _id:
            return 'I couldn\'t find your device.'

        metrics_names = metrics.split('.')
        _, filter = self.metric_filter(metrics_names)

        cal = parsedatetime.Calendar()
        if not period:
            period = '2 hours ago'
        past, _ = cal.parseDT(datetimeString=period, tzinfo=self.timezone)
        now = datetime.now(self.timezone)

        if past > now:
            return 'Hey, I can\'t predict your data into the future, your date has to be in the past and now your date is {}'.format(
                past)

        metrics_data = self.metrics.get(_id, past, now, filter)
        device, names = self.get_data(metrics_data)
        if not device.get('data'):
            text = (
                'It might be that your device is offline or has no metrics for `{}`.'
                .format(metrics) +
                'You can see what metrics are available by using `sdbot devices available {}`'
                .format(name))
            return text

        # creates file
        slack = Slacker(os.environ.get('SLACK_TOKEN'))
        slack.chat.post_message(
            self.msg['channel'],
            'Preparing the graphs for you this very moment',
            as_user=self.server.slack.server.username)

        filename = self.create_graph(device, now - past)

        attachment = [{
            'text': ('I brought you a graph for {} for the device `{}`'.format(
                ' '.join(names), name) +
                     '\n A graph is coming up in just a sec.'),
            'mrkdwn_in': ['text'],
            'color':
            COLOR
        }]

        slack.chat.post_message(self.msg['channel'],
                                '',
                                attachments=attachment,
                                as_user=self.server.slack.server.username)

        # uploads and sends the graph to channel
        file_response = slack.files.upload(filename,
                                           filename='{}.png'.format(name),
                                           channels=self.msg['channel'])
        # Deletes the graph file after upload so it doesn't litter the container
        os.remove(filename)

        return None  # We're sending information in function itself this time
예제 #18
0
class DataWrapper(object):
    def __init__(self, token, conf):
        self.token = token
        self.conf = conf
        self.historic_data = None
        client = ApiClient(self.token, timeout=(3.05, 18))
        self.device = Device(self.token)
        self.tag = Tag(self.token)
        self.metrics = Metrics(client)

        self._validation()

    def _validation(self):
        """Check the following
        - metric for every piece in infrastructure
        - That the configuration contains everything necessary.
        - check that there are no none values in any lists.
        """

    def _get_devices(self, infra_conf):
        """Takes the configuration part for each infrastructure"""
        raw_devices = self.device.list()

        if infra_conf.get('tag'):
            tags = self.tag.list()
            tag_id = [
                tag['_id'] for tag in tags if tag['name'] == infra_conf['tag']
            ]
            if not tag_id:
                available_tags = '\n'.join(
                    list(set([tag['name'] for tag in tags])))
                message = 'There is no tag with the name "{}". Try one of these: \n{}'.format(
                    infra_conf['tag'], available_tags)
                raise Exception(message)
            else:
                tag_id = tag_id[0]
            devices = [
                device for device in raw_devices
                if tag_id in device.get('tags', [])
            ]
            if not devices:
                available_tags = '\n'.join(
                    list(set([tag['name'] for tag in tags])))
                raise Exception(
                    'There is no device with this tag name. Try one of these: \n{}'
                    .format(available_tags))
        elif infra_conf.get('group'):
            group = infra_conf.get('group')
            devices = [
                device for device in raw_devices
                if group == device.get('group')
            ]
            if not devices:
                groups = set([
                    device['group'] for device in raw_devices
                    if not device['group'] is None
                ])
                groups = '\n'.join(list(groups))
                raise Exception(
                    'There is no device with this group name. The following groups are available:\n {}'
                    .format(groups))
        else:
            raise Exception(
                'You have to provide either group or tag for each part of your infrastructure.'
            )
        return devices

    def _merge_loadbalanced_data(self, data_points, points):
        max_length = len(data_points)
        for i, point in enumerate(points):
            if i < max_length:
                data_points[i] = data_points[i] + point
        return data_points

    def _get_metrics(self, metric, devices):
        """For all devices associated with the group or device"""
        metric = metric.split('.')
        metric_filter = self.metric_filter(metric)
        end = dt.datetime.now()
        start = end - timedelta(
            hours=self.conf['general'].get('timeframe', 24))
        data_entries = []
        for device in devices:
            data = self.metrics.get(device['_id'], start, end, metric_filter)
            data = self._data_node(data)
            time.sleep(0.3)
            if data['data']:
                data_entries.append(data)

        if not data_entries:
            metric = '.'.join(metric)
            # Append zero data to avoid zerodivison error
            data_entries.append({'data': [{'x': 0, 'y': 0}]})
            logging.warning(
                'No server in this group has any data on {}'.format(metric))
        return data_entries

    def _data_node(self, data, names=None):
        """Inputs the data from the metrics endpoints and returns
        the node that has contains the data + names of the metrics."""

        if not names:
            names = []
        for d in data:
            if d.get('data') or d.get('data') == []:
                names.append(d.get('name'))
                d['full_name'] = names
                return d
            else:
                names.append(d.get('name'))
                return self._data_node(d.get('tree'), names)

    def _get_data_points(self, cumulative, data_entries, multiplier):
        """Extract the singular points into a list and return a list of those points"""
        data_points = []
        for data in data_entries:
            points = [point['y'] * multiplier for point in data['data']]
            if cumulative and len(data_points) > 0:
                self._merge_loadbalanced_data(data_points, points)
            else:
                data_points.extend(points)
        return data_points

    def _round(self, number):
        rounding = self.conf['general'].get('round', 2)
        if rounding > 0:
            return round(number, rounding)
        else:
            return int(round(number, 0))

    def calc_average(self, data_points):
        return self._round(sum(data_points) / len(data_points))

    def calc_max(self, data_points):
        return self._round(max(data_points))

    def calc_min(self, data_points):
        return self._round(min(data_points))

    def calc_median(self, data_points):
        data_points.sort()
        start = len(data_points) // 2.0
        if len(data_points) % 2 > 0:
            result = (data_points[start] + data_points[start + 1]) / 2.0
        else:
            result = self._round(data_points[start] // 2.0)
        return result

    def calc_sum(self, data_points):
        return self._round(sum(data_points))

    def gather_data(self):
        """The main function that gathers all data and returns an updated
        configuration file that the index template can read"""

        infrastructure = self.conf['infrastructure']
        for infra_conf in infrastructure:
            logging.info('Gather data from {}...'.format(infra_conf['title']))
            devices = self._get_devices(infra_conf)
            for metric_conf in infra_conf['metrics']:
                metric = metric_conf.get('metrickey')
                cumulative = metric_conf.get('cumulative', True)
                # giving the option to show case static information in boxes
                if metric:
                    multiplier = metric_conf.get('multiplier', 1)
                    data_entries = self._get_metrics(metric, devices)
                    for method in metric_conf['calculation']:
                        data_points = self._get_data_points(
                            cumulative, data_entries, multiplier)
                        result = getattr(self,
                                         'calc_{}'.format(method))(data_points)
                        metric_conf['{}_stat'.format(method)] = result
        return self.conf

    def metric_filter(self, metrics, filter=None):
        """from a list of metrics ie ['cpuStats', 'CPUs', 'usr'] it constructs
        a dictionary that can be sent to the metrics endpoint for consumption"""

        metrics = list(metrics)
        if not filter:
            filter = {}
            filter[metrics.pop()] = 'all'
            return self.metric_filter(metrics, filter)
        else:
            try:
                metric = metrics.pop()
                dic = {metric: filter}
                return self.metric_filter(metrics, dic)
            except IndexError:
                return filter

    def available(self):
        """Assumes that all metrics are the same for a group or a tag"""
        infrastructure = self.conf['infrastructure']
        md = '# Available metrics for all your groups\n\n'

        for infra_conf in infrastructure:
            if infra_conf.get('group'):
                category = infra_conf['group']
            elif infra_conf.get('tag'):
                category = infra_conf['tags']
            else:
                raise Exception('You need to provide either a group or tag')
            logging.info('Gathering metrics from {}...'.format(category))
            devices = self._get_devices(infra_conf)
            device = devices[0]
            end = dt.datetime.now()
            start = end - timedelta(hours=2)
            available = self.metrics.available(device['_id'], start, end)
            metrics = self.flatten(available)
            try:
                md += '## {}\n'.format(infra_conf['title'])
            except KeyError:
                raise KeyError(
                    'Each section need a title, go on fill one in and try again.'
                )
            for metric in metrics:
                title = ' '.join([tup[0] for tup in metric])
                metric = '.'.join([tup[1] for tup in metric])
                entry = '##### {}\nmetrickey: {}\n\n'.format(title, metric)
                md += entry
        with codecs.open('available.md', 'w') as f:
            f.write(md)

    def flatten(self, lst):
        """Get all the keys when calling available"""
        for dct in lst:
            key = dct['key']
            name = dct['name']
            if 'tree' not in dct:
                yield [(name, key)]  # base case
            else:
                for result in self.flatten(dct["tree"]):  # recursive case
                    yield [(name, key)] + result