Example #1
0
 def test_human2bytes(self):
     self.assertEqual(human2bytes('1B'), 1)
     self.assertEqual(human2bytes('9K'), 9216)
     self.assertEqual(human2bytes('1129M'), 1183842304)
     self.assertEqual(human2bytes('67T'), 73667279060992)
     self.assertEqual(human2bytes('13P'), 14636698788954112)
     self.assertRaises(ValueError, human2bytes, '')
     self.assertRaises(ValueError, human2bytes, 'faoej')
     self.assertRaises(ValueError, human2bytes, '123KB')
     self.assertRaises(ValueError, human2bytes, '48')
     self.assertRaises(ValueError, human2bytes, '23V')
     self.assertRaises(TypeError, human2bytes, 234)
Example #2
0
 def test_human2bytes(self):
     self.assertEqual(human2bytes('1B'), 1)
     self.assertEqual(human2bytes('9K'), 9216)
     self.assertEqual(human2bytes('1129M'), 1183842304)
     self.assertEqual(human2bytes('67T'), 73667279060992)
     self.assertEqual(human2bytes('13P'), 14636698788954112)
     self.assertRaises(ValueError, human2bytes, '')
     self.assertRaises(ValueError, human2bytes, 'faoej')
     self.assertRaises(ValueError, human2bytes, '123KB')
     self.assertRaises(ValueError, human2bytes, '48')
     self.assertRaises(ValueError, human2bytes, '23V')
     self.assertRaises(TypeError, human2bytes, 234)
Example #3
0
    def __init__(self, *args, **config):
        super(ResourceWatcher, self).__init__(*args, **config)
        self.watcher = config.get("watcher", None)
        self.service = config.get("service", None)
        if self.service is not None:
            warnings.warn(
                "ResourceWatcher.service is deprecated "
                "please use ResourceWatcher.watcher instead.",
                category=DeprecationWarning)
            if self.watcher is None:
                self.watcher = self.service
        if self.watcher is None:
            self.statsd.stop()
            self.loop.close()
            raise NotImplementedError('watcher is mandatory for now.')

        self.max_cpu = float(config.get("max_cpu", 90))  # in %
        self.max_mem = config.get("max_mem")

        if self.max_mem is None:
            self.max_mem = 90.
            self._max_percent = True
        else:
            try:
                self.max_mem = float(self.max_mem)  # float -> %
                self._max_percent = True
            except ValueError:
                self.max_mem = human2bytes(self.max_mem)  # int -> absolute
                self._max_percent = False

        self.min_cpu = config.get("min_cpu")
        if self.min_cpu is not None:
            self.min_cpu = float(self.min_cpu)  # in %
        self.min_mem = config.get("min_mem")
        if self.min_mem is not None:
            try:
                self.min_mem = float(self.min_mem)  # float -> %
                self._min_percent = True
            except ValueError:
                self.min_mem = human2bytes(self.min_mem)  # int -> absolute
                self._min_percent = True
        self.health_threshold = float(config.get("health_threshold",
                                                 75))  # in %
        self.max_count = int(config.get("max_count", 3))

        self.process_children = to_bool(config.get("process_children", '0'))
        self.child_signal = int(config.get("child_signal", signal.SIGTERM))

        self._count_over_cpu = {}
        self._count_over_mem = {}
        self._count_under_cpu = {}
        self._count_under_mem = {}
        self._count_health = {}
    def __init__(self, *args, **config):
        super(ResourceWatcher, self).__init__(*args, **config)
        self.watcher = config.get("watcher", None)
        self.service = config.get("service", None)
        if self.service is not None:
            warnings.warn("ResourceWatcher.service is deprecated "
                          "please use ResourceWatcher.watcher instead.",
                          category=DeprecationWarning)
            if self.watcher is None:
                self.watcher = self.service
        if self.watcher is None:
            self.statsd.stop()
            self.loop.close()
            raise NotImplementedError('watcher is mandatory for now.')

        self.max_cpu = float(config.get("max_cpu", 90))     # in %
        self.max_mem = config.get("max_mem")

        if self.max_mem is None:
            self.max_mem = 90.
            self._max_percent = True
        else:
            try:
                self.max_mem = float(self.max_mem)          # float -> %
                self._max_percent = True
            except ValueError:
                self.max_mem = human2bytes(self.max_mem)    # int -> absolute
                self._max_percent = False

        self.min_cpu = config.get("min_cpu")
        if self.min_cpu is not None:
            self.min_cpu = float(self.min_cpu)              # in %
        self.min_mem = config.get("min_mem")
        if self.min_mem is not None:
            try:
                self.min_mem = float(self.min_mem)          # float -> %
                self._min_percent = True
            except ValueError:
                self.min_mem = human2bytes(self.min_mem)    # int -> absolute
                self._min_percent = True
        self.health_threshold = float(config.get("health_threshold",
                                      75))  # in %
        self.max_count = int(config.get("max_count", 3))

        self.process_children = to_bool(config.get("process_children", '0'))
        self.child_signal = int(config.get("child_signal", signal.SIGTERM))

        self._count_over_cpu = {}
        self._count_over_mem = {}
        self._count_under_cpu = {}
        self._count_under_mem = {}
        self._count_health = {}
Example #5
0
    def __init__(self, *args, **config):
        super(ResourceWatcher, self).__init__(*args, **config)

        # Watcher/service parameter
        self.watcher = config.get("watcher", None)
        self.service = config.get("service", None)
        if self.service is not None:
            warnings.warn("ResourceWatcher.service is deprecated "
                          "please use ResourceWatcher.watcher instead.",
                          category=DeprecationWarning)
            if self.watcher is None:
                self.watcher = self.service
        if self.watcher is None:
            self.statsd.stop()
            self.loop.close()
            raise NotImplementedError('watcher is mandatory for now.')

        # Memory/CPU parameters
        self.max_cpu = float(config.get("max_cpu", 90))     # in %
        self.max_mem = config.get("max_mem")
        if self.max_mem is None:
            self.max_mem = 90.
        else:
            try:
                self.max_mem = float(self.max_mem)          # float -> %
            except ValueError:
                self.max_mem = human2bytes(self.max_mem)    # int -> absolute
        self.min_cpu = config.get("min_cpu")
        if self.min_cpu is not None:
            self.min_cpu = float(self.min_cpu)              # in %
        self.min_mem = config.get("min_mem")
        if self.min_mem is not None:
            try:
                self.max_mem = float(self.min_mem)          # float -> %
            except ValueError:
                self.max_mem = human2bytes(self.min_mem)    # int -> absolute

        # Other parameters
        self.health_threshold = float(config.get("health_threshold",
                                      75))                  # in %
        self.max_count = int(config.get("max_count", 3))
        self.action = config.get("action", 'restart').lower()
        if self.action not in VALID_ACTIONS:
            raise ValueError(self.action)
        self.per_process = bool(config.get("per_process", False))
        if self.per_process and self.action in ['reload', 'restart']:
            raise NotImplementedError("You can't restart or reload a process.")
        if not self.per_process and self.action not in ['reload', 'restart']:
            raise NotImplementedError("You can't send a signal to a watcher.")

        self._monitors = {}
Example #6
0
    def look_after(self):
        self.storage.disk_usage(self.disk_usage())

        net_sent, net_recv = self.net_io()
        self.storage.net_sent(net_sent)
        self.storage.net_recv(net_recv)

        self.storage.net_connections(self.connections_established())
        self.storage.connections(self.connections())

        info = self.call("stats")
        if info["status"] == "error":
            return
        for name, stats in info['infos'].items():
            if name.startswith("plugin:"):
                # ignore plugins
                continue

            cpus = []
            mems = []
            mem_infos = []

            for sub_name, sub_info in stats.items():
                if isinstance(sub_info, dict):
                    cpu = [sub_info['cpu']]
                    mem = [sub_info['mem']]
                    inf = [human2bytes(sub_info['mem_info1'])]

                    for p in sub_info['children']:
                        cpu.append(p['cpu'])
                        mem.append(p['mem'])
                        inf.append(human2bytes(p['mem_info1']))

                    cpus.append(sum(cpu))
                    mems.append(sum(mem))
                    mem_infos.append(sum(inf))

                if not cpus:
                    # if there are only dead processes, we have an empty list
                    # and we can't measure it
                    continue

                self.storage.cpu_max(name, max(cpus))
                self.storage.cpu_sum(name, sum(cpus))
                self.storage.mem_pct_max(name, max(mems))
                self.storage.mem_pct_sum(name, sum(mems))
                self.storage.mem_max(name, max(mem_infos))
                self.storage.mem_sum(name, sum(mem_infos))
    def _collect_data(self, stats):
        data = {}
        cpus = []
        mems = []
        mems_abs = []

        for sub_info in stats.values():
            if isinstance(sub_info, dict):
                cpus.append(100 if sub_info['cpu'] == 'N/A' else
                            float(sub_info['cpu']))
                mems.append(100 if sub_info['mem'] == 'N/A' else
                            float(sub_info['mem']))
                mems_abs.append(0 if sub_info['mem_info1'] == 'N/A' else
                                human2bytes(sub_info['mem_info1']))

        if cpus:
            data['max_cpu'] = max(cpus)
            data['max_mem'] = max(mems)
            data['max_mem_abs'] = max(mems_abs)
            data['min_cpu'] = min(cpus)
            data['min_mem'] = min(mems)
            data['min_mem_abs'] = min(mems_abs)
        else:
            # we dont' have any process running. max = 0 then
            data['max_cpu'] = 0
            data['max_mem'] = 0
            data['min_cpu'] = 0
            data['min_mem'] = 0
            data['max_mem_abs'] = 0
            data['min_mem_abs'] = 0

        return data
Example #8
0
    def _collect_data(self, stats):
        data = {}
        cpus = []
        mems = []
        mems_abs = []

        for sub_info in stats.values():
            if isinstance(sub_info, dict):
                cpus.append(100 if sub_info['cpu'] ==
                            'N/A' else float(sub_info['cpu']))
                mems.append(100 if sub_info['mem'] ==
                            'N/A' else float(sub_info['mem']))
                mems_abs.append(0 if sub_info['mem_info1'] ==
                                'N/A' else human2bytes(sub_info['mem_info1']))

        if cpus:
            data['max_cpu'] = max(cpus)
            data['max_mem'] = max(mems)
            data['max_mem_abs'] = max(mems_abs)
            data['min_cpu'] = min(cpus)
            data['min_mem'] = min(mems)
            data['min_mem_abs'] = min(mems_abs)
        else:
            # we dont' have any process running. max = 0 then
            data['max_cpu'] = 0
            data['max_mem'] = 0
            data['min_cpu'] = 0
            data['min_mem'] = 0
            data['max_mem_abs'] = 0
            data['min_mem_abs'] = 0

        return data
Example #9
0
    def collect_stats(self):
        stats = self.call("stats", name=self.watcher)
        if stats["status"] == "error":
            self.statsd.increment("_resource_watcher.%s.error" % self.watcher)
            logger.error("Can't get stats of %s" % self.watcher)
            return
        stats = dict((k, v) for k, v in six.iteritems(stats['info']) if
                     type(v) == dict)

        # Convert absolute memory to bytes
        for item in stats:
            if stats[item]['mem_info1'] == 'N/A':
                stats[item]['mem_abs'] = 'N/A'
            else:
                stats[item]['mem_abs'] = human2bytes(stats[item]['mem_info1'])

        # Compute watcher stats if not in per_process mode
        if not self.per_process:
            stats[self.watcher] = defaultdict(lambda: 'N/A')
            for item in filter(lambda x: x != self.watcher, stats):
                for k in ['cpu', 'mem', 'mem_abs']:
                    if stats[item][k] != 'N/A':
                        if stats[self.watcher][k] == 'N/A':
                            stats[self.watcher][k] = stats[item][k]
                        else:
                            stats[self.watcher][k] += stats[item][k]
        return stats
Example #10
0
    def look_after(self):
        info = self.call("stats")
        if info["status"] == "error":
            self.statsd.increment("_stats.error")
            return

        for name, stats in info['infos'].items():
            if name.startswith("plugin:"):
                # ignore plugins
                continue

            cpus = []
            mems = []
            mem_infos = []

            for sub_name, sub_info in stats.items():
                if isinstance(sub_info, dict):
                    cpus.append(sub_info['cpu'])
                    mems.append(sub_info['mem'])
                    mem_infos.append(human2bytes(sub_info['mem_info1']))
                elif sub_name == "spawn_count":
                    # spawn_count info is in the same level as processes
                    # dict infos, so if spawn_count is given, take it and
                    # continue
                    self.statsd.gauge("_stats.%s.spawn_count" % name,
                                      sub_info)

            self.statsd.gauge("_stats.%s.watchers_num" % name, len(cpus))

            if not cpus:
                # if there are only dead processes, we have an empty list
                # and we can't measure it
                continue

            self.statsd.gauge("_stats.%s.cpu_max" % name, max(cpus))
            self.statsd.gauge("_stats.%s.cpu_sum" % name, sum(cpus))
            self.statsd.gauge("_stats.%s.mem_pct_max" % name, max(mems))
            self.statsd.gauge("_stats.%s.mem_pct_sum" % name, sum(mems))
            self.statsd.gauge("_stats.%s.mem_max" % name, max(mem_infos))
            self.statsd.gauge("_stats.%s.mem_sum" % name, sum(mem_infos))
Example #11
0
    def look_after(self):
        info = self.call("stats")
        if info["status"] == "error":
            self.statsd.increment("_stats.error")
            return

        for name, stats in info['infos'].items():
            if name.startswith("plugin:"):
                # ignore plugins
                continue

            cpus = []
            mems = []
            mem_infos = []

            for sub_name, sub_info in stats.items():
                if isinstance(sub_info, dict):
                    cpus.append(sub_info['cpu'])
                    mems.append(sub_info['mem'])
                    mem_infos.append(human2bytes(sub_info['mem_info1']))
                elif sub_name == "spawn_count":
                    # spawn_count info is in the same level as processes
                    # dict infos, so if spawn_count is given, take it and
                    # continue
                    self.statsd.gauge("_stats.%s.spawn_count" % name,
                                      sub_info)

            self.statsd.gauge("_stats.%s.watchers_num" % name, len(cpus))

            if not cpus:
                # if there are only dead processes, we have an empty list
                # and we can't measure it
                continue

            self.statsd.gauge("_stats.%s.cpu_max" % name, max(cpus))
            self.statsd.gauge("_stats.%s.cpu_sum" % name, sum(cpus))
            self.statsd.gauge("_stats.%s.mem_pct_max" % name, max(mems))
            self.statsd.gauge("_stats.%s.mem_pct_sum" % name, sum(mems))
            self.statsd.gauge("_stats.%s.mem_max" % name, max(mem_infos))
            self.statsd.gauge("_stats.%s.mem_sum" % name, sum(mem_infos))