def __init__(self, config=None, handlers=[], name=None, configfile=None):
super(BaseCollector, self).__init__(config, handlers, name, configfile)
self.cpu_collector = CPUCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.memory_collector = MemoryCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.loadavg_collector = LoadAverageCollector(
config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.network_collector = NetworkCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.diskusage_collector = DiskUsageCollector(
config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.diskspace_collector = DiskSpaceCollector(
config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.vmstat_collector = VMStatCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10
})
self.collector = CPUCollector(config, None)
@patch('__builtin__.open')
@patch('os.access', Mock(return_value=True))
@patch.object(Collector, 'publish')
def test_should_open_proc_stat(self, publish_mock, open_mock):
open_mock.return_value = StringIO('')
self.collector.collect()
open_mock.assert_called_once_with('/proc/stat')
@patch.object(Collector, 'publish')
def test_should_work_with_synthetic_data(self, publish_mock):
with patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 100 200 300 400 500 0 0 0 0 0'
))):
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
with patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 0 0 0 0 0'
))):
self.collector.collect()
self.assertPublishedMany(publish_mock, {
'total.idle': 4.0,
'total.iowait': 5.0,
'total.nice': 2.0,
'total.system': 3.0,
'total.user': 1.0
})
@patch.object(Collector, 'publish')
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath('proc_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
metrics = {
'total.idle': 2440.8,
'total.iowait': 0.2,
'total.nice': 0.0,
'total.system': 0.2,
'total.user': 0.4
}
self.setDocExample(collector=self.collector.__class__.__name__,
metrics=metrics,
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 1,
'normalize': True,
})
self.collector = CPUCollector(config, None)
self.num_cpu = 2
# first measurement
self.input_base = {
'user': 100,
'nice': 200,
'system': 300,
'idle': 400,
}
# second measurement
self.input_next = {
'user': 110,
'nice': 220,
'system': 330,
'idle': 440,
}
# expected increment, divided by number of CPUs
# for example, user should be 10/2 = 5
self.expected = {
'total.user': 5.0,
'total.nice': 10.0,
'total.system': 15.0,
'total.idle': 20.0,
}
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10
})
self.collector = CPUCollector(config, None)
@patch('__builtin__.open')
@patch('os.access', Mock(return_value=True))
@patch.object(Collector, 'publish')
def test_should_open_proc_stat(self, publish_mock, open_mock):
open_mock.return_value = StringIO('')
self.collector.collect()
open_mock.assert_called_once_with('/proc/stat')
@patch.object(Collector, 'publish')
def test_should_work_with_synthetic_data(self, publish_mock):
with patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 100 200 300 400 500 0 0 0 0 0'
))):
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
with patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 0 0 0 0 0'
))):
self.collector.collect()
self.assertPublishedMany(publish_mock, {
'total.idle': 4.0,
'total.iowait': 5.0,
'total.nice': 2.0,
'total.system': 3.0,
'total.user': 1.0
})
@patch.object(Collector, 'publish')
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath('proc_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
metrics = {
'total.idle': 2440.8,
'total.iowait': 0.2,
'total.nice': 0.0,
'total.system': 0.2,
'total.user': 0.4
}
self.setDocExample(collector=self.collector.__class__.__name__,
metrics=metrics,
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)
class BaseCollector(diamond.collector.Collector):
def __init__(self, config=None, handlers=[], name=None, configfile=None):
super(BaseCollector, self).__init__(config, handlers, name, configfile)
self.cpu_collector = CPUCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.memory_collector = MemoryCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.loadavg_collector = LoadAverageCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.network_collector = NetworkCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.diskusage_collector = DiskUsageCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.diskspace_collector = DiskSpaceCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.vmstat_collector = VMStatCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
def get_default_config_help(self):
config_help = super(BaseCollector, self).get_default_config_help()
config_help.update({
'simple': 'run simple mode on of its sub collectors',
})
return config_help
def get_default_config(self):
config = super(BaseCollector, self).get_default_config()
return config
def collect(self):
self.cpu_collector.collect()
self.memory_collector.collect()
self.loadavg_collector.collect()
self.network_collector.collect()
self.diskusage_collector.collect()
self.diskspace_collector.collect()
self.vmstat_collector.collect()
return True
class BaseCollector(diamond.collector.Collector):
def __init__(self, config=None, handlers=[], name=None, configfile=None):
super(BaseCollector, self).__init__(config, handlers, name, configfile)
self.cpu_collector = CPUCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.memory_collector = MemoryCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.loadavg_collector = LoadAverageCollector(
config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.network_collector = NetworkCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.diskusage_collector = DiskUsageCollector(
config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.diskspace_collector = DiskSpaceCollector(
config=self.config,
configfile=self.configfile,
handlers=self.handlers)
self.vmstat_collector = VMStatCollector(config=self.config,
configfile=self.configfile,
handlers=self.handlers)
def get_default_config_help(self):
config_help = super(BaseCollector, self).get_default_config_help()
config_help.update({
'simple':
'run simple mode on of its sub collectors',
})
return config_help
def get_default_config(self):
config = super(BaseCollector, self).get_default_config()
return config
def collect(self):
self.cpu_collector.collect()
self.memory_collector.collect()
self.loadavg_collector.collect()
self.network_collector.collect()
self.diskusage_collector.collect()
self.diskspace_collector.collect()
self.vmstat_collector.collect()
return True
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 1,
'normalize': True,
})
self.collector = CPUCollector(config, None)
self.num_cpu = 2
# first measurement
self.input_base = {
'user': 100,
'nice': 200,
'system': 300,
'idle': 400,
}
# second measurement
self.input_next = {
'user': 110,
'nice': 220,
'system': 330,
'idle': 440,
}
# expected increment, divided by number of CPUs
# for example, user should be 10/2 = 5
self.expected = {
'total.user': 5.0,
'total.nice': 10.0,
'total.system': 15.0,
'total.idle': 20.0,
}
def __init__(self, config=None, handlers=[], name=None, configfile=None):
super(BaseCollector, self).__init__(config, handlers, name, configfile)
self.cpu_collector = CPUCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.memory_collector = MemoryCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.loadavg_collector = LoadAverageCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.network_collector = NetworkCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.diskusage_collector = DiskUsageCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.diskspace_collector = DiskSpaceCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
self.vmstat_collector = VMStatCollector(config=self.config, configfile=self.configfile, handlers=self.handlers)
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config("CPUCollector", {"interval": 10})
self.collector = CPUCollector(config, None)
@patch("__builtin__.open")
@patch("os.access", Mock(return_value=True))
@patch.object(Collector, "publish")
def test_should_open_proc_stat(self, publish_mock, open_mock):
open_mock.return_value = StringIO("")
self.collector.collect()
open_mock.assert_called_once_with("/proc/stat")
@patch.object(Collector, "publish")
def test_should_work_with_synthetic_data(self, publish_mock):
with patch("__builtin__.open", Mock(return_value=StringIO("cpu 100 200 300 400 500 0 0 0 0 0"))):
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
with patch("__builtin__.open", Mock(return_value=StringIO("cpu 110 220 330 440 550 0 0 0 0 0"))):
self.collector.collect()
self.assertPublishedMany(
publish_mock,
{"total.idle": 4.0, "total.iowait": 5.0, "total.nice": 2.0, "total.system": 3.0, "total.user": 1.0},
)
@patch.object(Collector, "publish")
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath("proc_stat_1")
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath("proc_stat_2")
self.collector.collect()
metrics = {"total.idle": 2440.8, "total.iowait": 0.2, "total.nice": 0.0, "total.system": 0.2, "total.user": 0.4}
self.setDocExample(self.collector.__class__.__name__, metrics)
self.assertPublishedMany(publish_mock, metrics)
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 1,
'normalize': True,
})
self.collector = CPUCollector(config, None)
self.num_cpu = 2
# first measurement
self.input_base = {
'user': 100,
'nice': 200,
'system': 300,
'idle': 400,
}
# second measurement
self.input_next = {
'user': 110,
'nice': 220,
'system': 330,
'idle': 440,
}
self.expected = {
'cpu.total.user': 110,
'cpu.total.nice': 220,
'cpu.total.system': 330,
'cpu.total.idle': 440,
}
self.expected2 = {
'cpu.total.user': 55,
'cpu.total.nice': 110,
'cpu.total.system': 165,
'cpu.total.idle': 220,
}
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 1,
'normalize': True,
})
self.collector = CPUCollector(config, None)
self.num_cpu = 2
# first measurement
self.input_base = {
'user': 100,
'nice': 200,
'system': 300,
'idle': 400,
}
# second measurement
self.input_next = {
'user': 110,
'nice': 220,
'system': 330,
'idle': 440,
}
self.expected = {
'cpu.total.user': 110,
'cpu.total.nice': 220,
'cpu.total.system': 330,
'cpu.total.idle': 440,
}
self.expected2 = {
'cpu.total.user': 55,
'cpu.total.nice': 110,
'cpu.total.system': 165,
'cpu.total.idle': 220,
}
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
def test_import(self):
self.assertTrue(CPUCollector)
@patch('__builtin__.open')
@patch('os.access', Mock(return_value=True))
@patch.object(Collector, 'publish')
def test_should_open_proc_stat(self, publish_mock, open_mock):
CPUCollector.PROC = '/proc/stat'
open_mock.return_value = StringIO('')
self.collector.collect()
open_mock.assert_called_once_with('/proc/stat')
@patch.object(Collector, 'publish')
def test_should_work_with_synthetic_data(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 100 200 300 400 500 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {})
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {
'total.idle': 4.0,
'total.iowait': 5.0,
'total.nice': 2.0,
'total.system': 3.0,
'total.user': 1.0
})
@patch.object(Collector, 'publish')
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath('proc_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
metrics = {
'total.idle': 2440.8,
'total.iowait': 0.2,
'total.nice': 0.0,
'total.system': 0.2,
'total.user': 0.4
}
self.setDocExample(collector=self.collector.__class__.__name__,
metrics=metrics,
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_should_work_with_ec2_data(self, publish_mock):
self.collector.config['interval'] = 30
patch_open = patch('os.path.isdir', Mock(return_value=True))
patch_open.start()
CPUCollector.PROC = self.getFixturePath('ec2_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('ec2_stat_2')
self.collector.collect()
patch_open.stop()
metrics = {
'total.idle': 68.4,
'total.iowait': 0.6,
'total.nice': 0.0,
'total.system': 13.7,
'total.user': 16.666666666666668
}
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_473(self, publish_mock):
"""
No cpu value should ever be over 100
"""
self.collector.config['interval'] = 60
patch_open = patch('os.path.isdir', Mock(return_value=True))
patch_open.start()
CPUCollector.PROC = self.getFixturePath('473_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('473_2')
self.collector.collect()
patch_open.stop()
totals = {}
for call in publish_mock.mock_calls:
call = call[1]
if call[0][:6] == 'total.':
continue
if call[1] > 100:
raise ValueError("metric %s: %s should not be over 100!" % (
call[0], call[1]))
k = call[0][:4]
totals[k] = totals.get(k, 0) + call[1]
for t in totals:
# Allow rounding errors
if totals[t] >= 101:
raise ValueError(
"metric total for %s: %s should not be over 100!" % (
t, totals[t]))
class TestCPUCollectorNormalize(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 1,
'normalize': True,
})
self.collector = CPUCollector(config, None)
self.num_cpu = 2
# first measurement
self.input_base = {
'user': 100,
'nice': 200,
'system': 300,
'idle': 400,
}
# second measurement
self.input_next = {
'user': 110,
'nice': 220,
'system': 330,
'idle': 440,
}
# expected increment, divided by number of CPUs
# for example, user should be 10/2 = 5
self.expected = {
'total.user': 5.0,
'total.nice': 10.0,
'total.system': 15.0,
'total.idle': 20.0,
}
# convert an input dict with values to a string that might come from
# /proc/stat
def input_dict_to_proc_string(self, cpu_id, dict_):
return ("cpu%s %i %i %i %i 0 0 0 0 0 0" %
(cpu_id,
dict_['user'],
dict_['nice'],
dict_['system'],
dict_['idle'],
)
)
@patch.object(Collector, 'publish')
def test_should_work_proc_stat(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
"\n".join([self.input_dict_to_proc_string('', self.input_base),
self.input_dict_to_proc_string('0', self.input_base),
self.input_dict_to_proc_string('1', self.input_base),
])
)))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {})
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
"\n".join([self.input_dict_to_proc_string('', self.input_next),
self.input_dict_to_proc_string('0', self.input_next),
self.input_dict_to_proc_string('1', self.input_next),
])
)))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, self.expected)
@patch.object(Collector, 'publish')
@patch('cpu.os')
@patch('cpu.psutil')
def test_should_work_psutil(self, psutil_mock, os_mock, publish_mock):
os_mock.access.return_value = False
total = Mock(**self.input_base)
cpu_time = [Mock(**self.input_base),
Mock(**self.input_base),
]
psutil_mock.cpu_times.side_effect = [cpu_time, total]
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
total = Mock(**self.input_next)
cpu_time = [Mock(**self.input_next),
Mock(**self.input_next),
]
psutil_mock.cpu_times.side_effect = [cpu_time, total]
self.collector.collect()
self.assertPublishedMany(publish_mock, self.expected)
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
def test_import(self):
self.assertTrue(CPUCollector)
@patch('__builtin__.open')
@patch('os.access', Mock(return_value=True))
@patch.object(Collector, 'publish')
def test_should_open_proc_stat(self, publish_mock, open_mock):
CPUCollector.PROC = '/proc/stat'
open_mock.return_value = StringIO('')
self.collector.collect()
open_mock.assert_called_once_with('/proc/stat')
@patch.object(Collector, 'publish')
def test_should_work_with_synthetic_data(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 100 200 300 400 500 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {})
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {
'total.idle': 4.0,
'total.iowait': 5.0,
'total.nice': 2.0,
'total.system': 3.0,
'total.user': 1.0
})
@patch.object(Collector, 'publish')
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath('proc_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
metrics = {
'total.idle': 2440.8,
'total.iowait': 0.2,
'total.nice': 0.0,
'total.system': 0.2,
'total.user': 0.4
}
self.setDocExample(collector=self.collector.__class__.__name__,
metrics=metrics,
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_should_work_with_ec2_data(self, publish_mock):
self.collector.config['interval'] = 30
patch_open = patch('os.path.isdir', Mock(return_value=True))
patch_open.start()
CPUCollector.PROC = self.getFixturePath('ec2_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('ec2_stat_2')
self.collector.collect()
patch_open.stop()
metrics = {
'total.idle': 68.4,
'total.iowait': 0.6,
'total.nice': 0.0,
'total.system': 13.7,
'total.user': 16.666666666666668
}
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_473(self, publish_mock):
"""
No cpu value should ever be over 100
"""
self.collector.config['interval'] = 60
patch_open = patch('os.path.isdir', Mock(return_value=True))
patch_open.start()
CPUCollector.PROC = self.getFixturePath('473_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('473_2')
self.collector.collect()
patch_open.stop()
totals = {}
for call in publish_mock.mock_calls:
call = call[1]
if call[0][:6] == 'total.':
continue
if call[1] > 100:
raise ValueError("metric %s: %s should not be over 100!" % (
call[0], call[1]))
k = call[0][:4]
totals[k] = totals.get(k, 0) + call[1]
for t in totals:
# Allow rounding errors
if totals[t] >= 101:
raise ValueError(
"metric total for %s: %s should not be over 100!" % (
t, totals[t]))
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
def test_import(self):
self.assertTrue(CPUCollector)
@patch('__builtin__.open')
@patch('os.access', Mock(return_value=True))
@patch.object(Collector, 'publish')
def test_should_open_proc_stat(self, publish_mock, open_mock):
CPUCollector.PROC = '/proc/stat'
open_mock.return_value = StringIO('')
self.collector.collect()
open_mock.assert_called_once_with('/proc/stat')
@patch.object(Collector, 'publish')
def test_should_work_with_synthetic_data(self, publish_mock):
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO('cpu 100 200 300 400 500 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {})
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO('cpu 110 220 330 440 550 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(
publish_mock, {
'cpu.total.idle': 440,
'cpu.total.iowait': 550,
'cpu.total.nice': 220,
'cpu.total.system': 330,
'cpu.total.user': 110
})
@patch.object(Collector, 'publish')
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath('proc_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
metrics = {
'cpu.total.idle': 3925832001,
'cpu.total.iowait': 575306,
'cpu.total.nice': 1104382,
'cpu.total.system': 8454154,
'cpu.total.user': 29055791
}
self.setDocExample(collector=self.collector.__class__.__name__,
metrics=metrics,
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_should_work_with_ec2_data(self, publish_mock):
self.collector.config['interval'] = 30
self.collector.config['xenfix'] = False
patch_open = patch('os.path.isdir', Mock(return_value=True))
patch_open.start()
CPUCollector.PROC = self.getFixturePath('ec2_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('ec2_stat_2')
self.collector.collect()
patch_open.stop()
metrics = {
'cpu.total.idle': 2806608501,
'cpu.total.iowait': 13567144,
'cpu.total.nice': 15545,
'cpu.total.system': 170762788,
'cpu.total.user': 243646997
}
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_total_metrics_enable_aggregation_false(self, publish_mock):
self.collector.config['enableAggregation'] = False
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
publishedMetrics = {
'cpu.total.nice': 1104382,
'cpu.total.irq': 3,
'cpu.total.softirq': 59032,
'cpu.total.user': 29055791
}
unpublishedMetrics = {
'cpu.total.user_mode': 30160173,
'cpu.total.irq_softirq': 59035
}
self.assertPublishedMany(publish_mock, publishedMetrics)
self.collector.collect()
self.assertUnpublishedMany(publish_mock, unpublishedMetrics)
@patch.object(Collector, 'publish')
def test_total_metrics_enable_aggregation_true(self, publish_mock):
self.collector.config['enableAggregation'] = True
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
publishedMetrics = {
'cpu.total.nice': 1104382,
'cpu.total.irq': 3,
'cpu.total.softirq': 59032,
'cpu.total.user': 29055791,
'cpu.total.user_mode': 30160173,
'cpu.total.irq_softirq': 59035
}
self.assertPublishedMany(publish_mock, publishedMetrics)
@patch.object(Collector, 'publish')
def test_total_metrics_enable_aggregation_true_blacklist(
self, publish_mock):
self.collector.config['enableAggregation'] = True
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
publishedMetrics = {
'cpu.total.nice': 1104382,
'cpu.total.irq': 3,
'cpu.total.softirq': 59032,
'cpu.total.user': 29055791,
'cpu.total.user_mode': 30160173,
'cpu.total.irq_softirq': 59035
}
self.assertPublishedMany(publish_mock, publishedMetrics)
@patch.object(Collector, 'publish')
def test_core_metrics_enable_aggregation_false(self, publish_mock):
self.collector.config['enableAggregation'] = False
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
publishedMetrics = {
'cpu.nice': 220,
'cpu.irq': 660,
'cpu.softirq': 770,
'cpu.user': 110
}
self.assertPublishedMany(publish_mock, publishedMetrics)
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
unpublishedMetrics = {'cpu.user_mode': 330, 'cpu.irq_softirq': 1430}
self.assertUnpublishedMany(publish_mock, unpublishedMetrics)
@patch.object(Collector, 'publish')
def test_core_metrics_enable_aggregation_true(self, publish_mock):
self.collector.config['enableAggregation'] = True
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
publishedMetrics = {
'cpu.nice': 220,
'cpu.irq': 660,
'cpu.softirq': 770,
'cpu.user': 110,
'cpu.user_mode': 330,
'cpu.irq_softirq': 1430
}
self.assertPublishedMany(publish_mock, publishedMetrics)
class TestCPUCollectorNormalize(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 1,
'normalize': True,
})
self.collector = CPUCollector(config, None)
self.num_cpu = 2
# first measurement
self.input_base = {
'user': 100,
'nice': 200,
'system': 300,
'idle': 400,
}
# second measurement
self.input_next = {
'user': 110,
'nice': 220,
'system': 330,
'idle': 440,
}
# expected increment, divided by number of CPUs
# for example, user should be 10/2 = 5
self.expected = {
'total.user': 5.0,
'total.nice': 10.0,
'total.system': 15.0,
'total.idle': 20.0,
}
# convert an input dict with values to a string that might come from
# /proc/stat
def input_dict_to_proc_string(self, cpu_id, dict_):
return ("cpu%s %i %i %i %i 0 0 0 0 0 0" %
(cpu_id,
dict_['user'],
dict_['nice'],
dict_['system'],
dict_['idle'],
)
)
@patch.object(Collector, 'publish')
def test_should_work_proc_stat(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
"\n".join([self.input_dict_to_proc_string('', self.input_base),
self.input_dict_to_proc_string('0', self.input_base),
self.input_dict_to_proc_string('1', self.input_base),
])
)))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {})
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
"\n".join([self.input_dict_to_proc_string('', self.input_next),
self.input_dict_to_proc_string('0', self.input_next),
self.input_dict_to_proc_string('1', self.input_next),
])
)))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, self.expected)
@patch.object(Collector, 'publish')
@patch('cpu.os')
@patch('cpu.psutil')
def test_should_work_psutil(self, psutil_mock, os_mock, publish_mock):
os_mock.access.return_value = False
total = Mock(**self.input_base)
cpu_time = [Mock(**self.input_base),
Mock(**self.input_base),
]
psutil_mock.cpu_times.side_effect = [cpu_time, total]
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
total = Mock(**self.input_next)
cpu_time = [Mock(**self.input_next),
Mock(**self.input_next),
]
psutil_mock.cpu_times.side_effect = [cpu_time, total]
self.collector.collect()
self.assertPublishedMany(publish_mock, self.expected)
class TestCPUCollectorDimensions(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
@patch.object(Collector, 'flush')
def test_core_dimension_core_metrics(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in ['cpu.user', 'cpu.idle', 'cpu.nice', 'cpu.softirq']:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertTrue(metrics[0]['dimensions'].has_key('core'))
@patch.object(Collector, 'flush')
def test_core_dimension_total_metrics(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in ['cpu.total.user', 'cpu.total.idle', 'cpu.total.nice', 'cpu.total.softirq']:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertFalse(metrics[0]['dimensions'].has_key('core'))
@patch.object(Collector, 'flush')
def test_core_dimension_core_metrics_aggregated(self, publish_mock):
self.collector.config['enableAggregation'] = True
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in ['cpu.user_mode', 'cpu.idle', 'cpu.nice', 'cpu.irq_softirq']:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertTrue(metrics[0]['dimensions'].has_key('core'))
@patch.object(Collector, 'flush')
def test_core_dimension_total_metrics_aggregated(self, publish_mock):
self.collector.config['enableAggregation'] = True
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in ['cpu.total.user_mode', 'cpu.total.idle', 'cpu.total.nice', 'cpu.total.irq_softirq']:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertFalse(metrics[0]['dimensions'].has_key('core'))
def setUp(self):
config = get_collector_config('CPUCollector', {'interval': 10})
self.collector = CPUCollector(config, None)
class TestCPUCollectorDimensions(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
@patch.object(Collector, 'flush')
def test_core_dimension_core_metrics(self, publish_mock):
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in ['cpu.user', 'cpu.idle', 'cpu.nice', 'cpu.softirq']:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertTrue(metrics[0]['dimensions'].has_key('core'))
@patch.object(Collector, 'flush')
def test_core_dimension_total_metrics(self, publish_mock):
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in [
'cpu.total.user', 'cpu.total.idle', 'cpu.total.nice',
'cpu.total.softirq'
]:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertFalse(metrics[0]['dimensions'].has_key('core'))
@patch.object(Collector, 'flush')
def test_core_dimension_core_metrics_aggregated(self, publish_mock):
self.collector.config['enableAggregation'] = True
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in [
'cpu.user_mode', 'cpu.idle', 'cpu.nice', 'cpu.irq_softirq'
]:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertTrue(metrics[0]['dimensions'].has_key('core'))
@patch.object(Collector, 'flush')
def test_core_dimension_total_metrics_aggregated(self, publish_mock):
self.collector.config['enableAggregation'] = True
patch_open = patch(
'__builtin__.open',
Mock(return_value=StringIO(
'cpu 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
for metric_name in [
'cpu.total.user_mode', 'cpu.total.idle', 'cpu.total.nice',
'cpu.total.irq_softirq'
]:
metrics = find_metric(self.collector.payload, metric_name)
self.assertEqual(len(metrics), 1)
self.assertFalse(metrics[0]['dimensions'].has_key('core'))
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10
})
self.collector = CPUCollector(config, None)
def setUp(self):
config = get_collector_config("CPUCollector", {"interval": 10})
self.collector = CPUCollector(config, None)
class TestCPUCollector(CollectorTestCase):
def setUp(self):
config = get_collector_config('CPUCollector', {
'interval': 10,
'normalize': False
})
self.collector = CPUCollector(config, None)
def test_import(self):
self.assertTrue(CPUCollector)
@patch('__builtin__.open')
@patch('os.access', Mock(return_value=True))
@patch.object(Collector, 'publish')
def test_should_open_proc_stat(self, publish_mock, open_mock):
CPUCollector.PROC = '/proc/stat'
open_mock.return_value = StringIO('')
self.collector.collect()
open_mock.assert_called_once_with('/proc/stat')
@patch.object(Collector, 'publish')
def test_should_work_with_synthetic_data(self, publish_mock):
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 100 200 300 400 500 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {})
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu 110 220 330 440 550 0 0 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
self.assertPublishedMany(publish_mock, {
'cpu.total.idle': 440,
'cpu.total.iowait': 550,
'cpu.total.nice': 220,
'cpu.total.system': 330,
'cpu.total.user': 110
})
@patch.object(Collector, 'publish')
def test_should_work_with_real_data(self, publish_mock):
CPUCollector.PROC = self.getFixturePath('proc_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
metrics = {
'cpu.total.idle': 3925832001,
'cpu.total.iowait': 575306,
'cpu.total.nice': 1104382,
'cpu.total.system': 8454154,
'cpu.total.user': 29055791
}
self.setDocExample(collector=self.collector.__class__.__name__,
metrics=metrics,
defaultpath=self.collector.config['path'])
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_should_work_with_ec2_data(self, publish_mock):
self.collector.config['interval'] = 30
self.collector.config['xenfix'] = False
patch_open = patch('os.path.isdir', Mock(return_value=True))
patch_open.start()
CPUCollector.PROC = self.getFixturePath('ec2_stat_1')
self.collector.collect()
self.assertPublishedMany(publish_mock, {})
CPUCollector.PROC = self.getFixturePath('ec2_stat_2')
self.collector.collect()
patch_open.stop()
metrics = {
'cpu.total.idle': 2806608501,
'cpu.total.iowait': 13567144,
'cpu.total.nice': 15545,
'cpu.total.system': 170762788,
'cpu.total.user': 243646997
}
self.assertPublishedMany(publish_mock, metrics)
@patch.object(Collector, 'publish')
def test_total_metrics_enable_aggregation_false(self, publish_mock):
self.collector.config['enableAggregation'] = False
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
publishedMetrics = {
'cpu.total.nice': 1104382,
'cpu.total.irq': 3,
'cpu.total.softirq': 59032,
'cpu.total.user': 29055791
}
unpublishedMetrics = {
'cpu.total.user_mode': 30160173,
'cpu.total.irq_softirq': 59035
}
self.assertPublishedMany(publish_mock, publishedMetrics)
self.collector.collect()
self.assertUnpublishedMany(publish_mock, unpublishedMetrics)
@patch.object(Collector, 'publish')
def test_total_metrics_enable_aggregation_true(self, publish_mock):
self.collector.config['enableAggregation'] = True
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
publishedMetrics = {
'cpu.total.nice': 1104382,
'cpu.total.irq': 3,
'cpu.total.softirq': 59032,
'cpu.total.user': 29055791,
'cpu.total.user_mode': 30160173,
'cpu.total.irq_softirq': 59035
}
self.assertPublishedMany(publish_mock, publishedMetrics)
@patch.object(Collector, 'publish')
def test_total_metrics_enable_aggregation_true_blacklist(self, publish_mock):
self.collector.config['enableAggregation'] = True
CPUCollector.PROC = self.getFixturePath('proc_stat_2')
self.collector.collect()
publishedMetrics = {
'cpu.total.nice': 1104382,
'cpu.total.irq': 3,
'cpu.total.softirq': 59032,
'cpu.total.user': 29055791,
'cpu.total.user_mode': 30160173,
'cpu.total.irq_softirq': 59035
}
self.assertPublishedMany(publish_mock, publishedMetrics)
@patch.object(Collector, 'publish')
def test_core_metrics_enable_aggregation_false(self, publish_mock):
self.collector.config['enableAggregation'] = False
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
publishedMetrics = {
'cpu.nice': 220,
'cpu.irq': 660,
'cpu.softirq': 770,
'cpu.user': 110
}
self.assertPublishedMany(publish_mock, publishedMetrics)
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
unpublishedMetrics = {
'cpu.user_mode': 330,
'cpu.irq_softirq': 1430
}
self.assertUnpublishedMany(publish_mock, unpublishedMetrics)
@patch.object(Collector, 'publish')
def test_core_metrics_enable_aggregation_true(self, publish_mock):
self.collector.config['enableAggregation'] = True
patch_open = patch('__builtin__.open', Mock(return_value=StringIO(
'cpu0 110 220 330 440 550 660 770 0 0 0')))
patch_open.start()
self.collector.collect()
patch_open.stop()
publishedMetrics = {
'cpu.nice': 220,
'cpu.irq': 660,
'cpu.softirq': 770,
'cpu.user': 110,
'cpu.user_mode': 330,
'cpu.irq_softirq': 1430
}
self.assertPublishedMany(publish_mock, publishedMetrics)
