def test_human2bytes(self):
self.assertEqual(human2bytes('1B'), 1)
self.assertEqual(human2bytes('9K'), 9216)
self.assertEqual(human2bytes('1129M'), 1183842304)
self.assertEqual(human2bytes('67T'), 73667279060992)
self.assertEqual(human2bytes('13P'), 14636698788954112)
self.assertRaises(ValueError, human2bytes, '')
self.assertRaises(ValueError, human2bytes, 'faoej')
self.assertRaises(ValueError, human2bytes, '123KB')
self.assertRaises(ValueError, human2bytes, '48')
self.assertRaises(ValueError, human2bytes, '23V')
self.assertRaises(TypeError, human2bytes, 234)
def test_human2bytes(self):
self.assertEqual(human2bytes('1B'), 1)
self.assertEqual(human2bytes('9K'), 9216)
self.assertEqual(human2bytes('1129M'), 1183842304)
self.assertEqual(human2bytes('67T'), 73667279060992)
self.assertEqual(human2bytes('13P'), 14636698788954112)
self.assertRaises(ValueError, human2bytes, '')
self.assertRaises(ValueError, human2bytes, 'faoej')
self.assertRaises(ValueError, human2bytes, '123KB')
self.assertRaises(ValueError, human2bytes, '48')
self.assertRaises(ValueError, human2bytes, '23V')
self.assertRaises(TypeError, human2bytes, 234)
def __init__(self, *args, **config):
super(ResourceWatcher, self).__init__(*args, **config)
self.watcher = config.get("watcher", None)
self.service = config.get("service", None)
if self.service is not None:
warnings.warn(
"ResourceWatcher.service is deprecated "
"please use ResourceWatcher.watcher instead.",
category=DeprecationWarning)
if self.watcher is None:
self.watcher = self.service
if self.watcher is None:
self.statsd.stop()
self.loop.close()
raise NotImplementedError('watcher is mandatory for now.')
self.max_cpu = float(config.get("max_cpu", 90)) # in %
self.max_mem = config.get("max_mem")
if self.max_mem is None:
self.max_mem = 90.
self._max_percent = True
else:
try:
self.max_mem = float(self.max_mem) # float -> %
self._max_percent = True
except ValueError:
self.max_mem = human2bytes(self.max_mem) # int -> absolute
self._max_percent = False
self.min_cpu = config.get("min_cpu")
if self.min_cpu is not None:
self.min_cpu = float(self.min_cpu) # in %
self.min_mem = config.get("min_mem")
if self.min_mem is not None:
try:
self.min_mem = float(self.min_mem) # float -> %
self._min_percent = True
except ValueError:
self.min_mem = human2bytes(self.min_mem) # int -> absolute
self._min_percent = True
self.health_threshold = float(config.get("health_threshold",
75)) # in %
self.max_count = int(config.get("max_count", 3))
self.process_children = to_bool(config.get("process_children", '0'))
self.child_signal = int(config.get("child_signal", signal.SIGTERM))
self._count_over_cpu = {}
self._count_over_mem = {}
self._count_under_cpu = {}
self._count_under_mem = {}
self._count_health = {}
def __init__(self, *args, **config):
super(ResourceWatcher, self).__init__(*args, **config)
self.watcher = config.get("watcher", None)
self.service = config.get("service", None)
if self.service is not None:
warnings.warn("ResourceWatcher.service is deprecated "
"please use ResourceWatcher.watcher instead.",
category=DeprecationWarning)
if self.watcher is None:
self.watcher = self.service
if self.watcher is None:
self.statsd.stop()
self.loop.close()
raise NotImplementedError('watcher is mandatory for now.')
self.max_cpu = float(config.get("max_cpu", 90)) # in %
self.max_mem = config.get("max_mem")
if self.max_mem is None:
self.max_mem = 90.
self._max_percent = True
else:
try:
self.max_mem = float(self.max_mem) # float -> %
self._max_percent = True
except ValueError:
self.max_mem = human2bytes(self.max_mem) # int -> absolute
self._max_percent = False
self.min_cpu = config.get("min_cpu")
if self.min_cpu is not None:
self.min_cpu = float(self.min_cpu) # in %
self.min_mem = config.get("min_mem")
if self.min_mem is not None:
try:
self.min_mem = float(self.min_mem) # float -> %
self._min_percent = True
except ValueError:
self.min_mem = human2bytes(self.min_mem) # int -> absolute
self._min_percent = True
self.health_threshold = float(config.get("health_threshold",
75)) # in %
self.max_count = int(config.get("max_count", 3))
self.process_children = to_bool(config.get("process_children", '0'))
self.child_signal = int(config.get("child_signal", signal.SIGTERM))
self._count_over_cpu = {}
self._count_over_mem = {}
self._count_under_cpu = {}
self._count_under_mem = {}
self._count_health = {}
def __init__(self, *args, **config):
super(ResourceWatcher, self).__init__(*args, **config)
# Watcher/service parameter
self.watcher = config.get("watcher", None)
self.service = config.get("service", None)
if self.service is not None:
warnings.warn("ResourceWatcher.service is deprecated "
"please use ResourceWatcher.watcher instead.",
category=DeprecationWarning)
if self.watcher is None:
self.watcher = self.service
if self.watcher is None:
self.statsd.stop()
self.loop.close()
raise NotImplementedError('watcher is mandatory for now.')
# Memory/CPU parameters
self.max_cpu = float(config.get("max_cpu", 90)) # in %
self.max_mem = config.get("max_mem")
if self.max_mem is None:
self.max_mem = 90.
else:
try:
self.max_mem = float(self.max_mem) # float -> %
except ValueError:
self.max_mem = human2bytes(self.max_mem) # int -> absolute
self.min_cpu = config.get("min_cpu")
if self.min_cpu is not None:
self.min_cpu = float(self.min_cpu) # in %
self.min_mem = config.get("min_mem")
if self.min_mem is not None:
try:
self.max_mem = float(self.min_mem) # float -> %
except ValueError:
self.max_mem = human2bytes(self.min_mem) # int -> absolute
# Other parameters
self.health_threshold = float(config.get("health_threshold",
75)) # in %
self.max_count = int(config.get("max_count", 3))
self.action = config.get("action", 'restart').lower()
if self.action not in VALID_ACTIONS:
raise ValueError(self.action)
self.per_process = bool(config.get("per_process", False))
if self.per_process and self.action in ['reload', 'restart']:
raise NotImplementedError("You can't restart or reload a process.")
if not self.per_process and self.action not in ['reload', 'restart']:
raise NotImplementedError("You can't send a signal to a watcher.")
self._monitors = {}
def look_after(self):
self.storage.disk_usage(self.disk_usage())
net_sent, net_recv = self.net_io()
self.storage.net_sent(net_sent)
self.storage.net_recv(net_recv)
self.storage.net_connections(self.connections_established())
self.storage.connections(self.connections())
info = self.call("stats")
if info["status"] == "error":
return
for name, stats in info['infos'].items():
if name.startswith("plugin:"):
# ignore plugins
continue
cpus = []
mems = []
mem_infos = []
for sub_name, sub_info in stats.items():
if isinstance(sub_info, dict):
cpu = [sub_info['cpu']]
mem = [sub_info['mem']]
inf = [human2bytes(sub_info['mem_info1'])]
for p in sub_info['children']:
cpu.append(p['cpu'])
mem.append(p['mem'])
inf.append(human2bytes(p['mem_info1']))
cpus.append(sum(cpu))
mems.append(sum(mem))
mem_infos.append(sum(inf))
if not cpus:
# if there are only dead processes, we have an empty list
# and we can't measure it
continue
self.storage.cpu_max(name, max(cpus))
self.storage.cpu_sum(name, sum(cpus))
self.storage.mem_pct_max(name, max(mems))
self.storage.mem_pct_sum(name, sum(mems))
self.storage.mem_max(name, max(mem_infos))
self.storage.mem_sum(name, sum(mem_infos))
def _collect_data(self, stats):
data = {}
cpus = []
mems = []
mems_abs = []
for sub_info in stats.values():
if isinstance(sub_info, dict):
cpus.append(100 if sub_info['cpu'] == 'N/A' else
float(sub_info['cpu']))
mems.append(100 if sub_info['mem'] == 'N/A' else
float(sub_info['mem']))
mems_abs.append(0 if sub_info['mem_info1'] == 'N/A' else
human2bytes(sub_info['mem_info1']))
if cpus:
data['max_cpu'] = max(cpus)
data['max_mem'] = max(mems)
data['max_mem_abs'] = max(mems_abs)
data['min_cpu'] = min(cpus)
data['min_mem'] = min(mems)
data['min_mem_abs'] = min(mems_abs)
else:
# we dont' have any process running. max = 0 then
data['max_cpu'] = 0
data['max_mem'] = 0
data['min_cpu'] = 0
data['min_mem'] = 0
data['max_mem_abs'] = 0
data['min_mem_abs'] = 0
return data
def _collect_data(self, stats):
data = {}
cpus = []
mems = []
mems_abs = []
for sub_info in stats.values():
if isinstance(sub_info, dict):
cpus.append(100 if sub_info['cpu'] ==
'N/A' else float(sub_info['cpu']))
mems.append(100 if sub_info['mem'] ==
'N/A' else float(sub_info['mem']))
mems_abs.append(0 if sub_info['mem_info1'] ==
'N/A' else human2bytes(sub_info['mem_info1']))
if cpus:
data['max_cpu'] = max(cpus)
data['max_mem'] = max(mems)
data['max_mem_abs'] = max(mems_abs)
data['min_cpu'] = min(cpus)
data['min_mem'] = min(mems)
data['min_mem_abs'] = min(mems_abs)
else:
# we dont' have any process running. max = 0 then
data['max_cpu'] = 0
data['max_mem'] = 0
data['min_cpu'] = 0
data['min_mem'] = 0
data['max_mem_abs'] = 0
data['min_mem_abs'] = 0
return data
def collect_stats(self):
stats = self.call("stats", name=self.watcher)
if stats["status"] == "error":
self.statsd.increment("_resource_watcher.%s.error" % self.watcher)
logger.error("Can't get stats of %s" % self.watcher)
return
stats = dict((k, v) for k, v in six.iteritems(stats['info']) if
type(v) == dict)
# Convert absolute memory to bytes
for item in stats:
if stats[item]['mem_info1'] == 'N/A':
stats[item]['mem_abs'] = 'N/A'
else:
stats[item]['mem_abs'] = human2bytes(stats[item]['mem_info1'])
# Compute watcher stats if not in per_process mode
if not self.per_process:
stats[self.watcher] = defaultdict(lambda: 'N/A')
for item in filter(lambda x: x != self.watcher, stats):
for k in ['cpu', 'mem', 'mem_abs']:
if stats[item][k] != 'N/A':
if stats[self.watcher][k] == 'N/A':
stats[self.watcher][k] = stats[item][k]
else:
stats[self.watcher][k] += stats[item][k]
return stats
def look_after(self):
info = self.call("stats")
if info["status"] == "error":
self.statsd.increment("_stats.error")
return
for name, stats in info['infos'].items():
if name.startswith("plugin:"):
# ignore plugins
continue
cpus = []
mems = []
mem_infos = []
for sub_name, sub_info in stats.items():
if isinstance(sub_info, dict):
cpus.append(sub_info['cpu'])
mems.append(sub_info['mem'])
mem_infos.append(human2bytes(sub_info['mem_info1']))
elif sub_name == "spawn_count":
# spawn_count info is in the same level as processes
# dict infos, so if spawn_count is given, take it and
# continue
self.statsd.gauge("_stats.%s.spawn_count" % name,
sub_info)
self.statsd.gauge("_stats.%s.watchers_num" % name, len(cpus))
if not cpus:
# if there are only dead processes, we have an empty list
# and we can't measure it
continue
self.statsd.gauge("_stats.%s.cpu_max" % name, max(cpus))
self.statsd.gauge("_stats.%s.cpu_sum" % name, sum(cpus))
self.statsd.gauge("_stats.%s.mem_pct_max" % name, max(mems))
self.statsd.gauge("_stats.%s.mem_pct_sum" % name, sum(mems))
self.statsd.gauge("_stats.%s.mem_max" % name, max(mem_infos))
self.statsd.gauge("_stats.%s.mem_sum" % name, sum(mem_infos))
def look_after(self):
info = self.call("stats")
if info["status"] == "error":
self.statsd.increment("_stats.error")
return
for name, stats in info['infos'].items():
if name.startswith("plugin:"):
# ignore plugins
continue
cpus = []
mems = []
mem_infos = []
for sub_name, sub_info in stats.items():
if isinstance(sub_info, dict):
cpus.append(sub_info['cpu'])
mems.append(sub_info['mem'])
mem_infos.append(human2bytes(sub_info['mem_info1']))
elif sub_name == "spawn_count":
# spawn_count info is in the same level as processes
# dict infos, so if spawn_count is given, take it and
# continue
self.statsd.gauge("_stats.%s.spawn_count" % name,
sub_info)
self.statsd.gauge("_stats.%s.watchers_num" % name, len(cpus))
if not cpus:
# if there are only dead processes, we have an empty list
# and we can't measure it
continue
self.statsd.gauge("_stats.%s.cpu_max" % name, max(cpus))
self.statsd.gauge("_stats.%s.cpu_sum" % name, sum(cpus))
self.statsd.gauge("_stats.%s.mem_pct_max" % name, max(mems))
self.statsd.gauge("_stats.%s.mem_pct_sum" % name, sum(mems))
self.statsd.gauge("_stats.%s.mem_max" % name, max(mem_infos))
self.statsd.gauge("_stats.%s.mem_sum" % name, sum(mem_infos))
