def test_polling_exception(self, LOG):
source_name = 'test_pollingexception'
self.pipeline_cfg = [
{
'name': source_name,
'interval': 10,
'counters': ['testpollingexception'],
'resources': ['test://'] if self.source_resources else [],
'transformers': [],
'publishers': ["test"],
},
]
self.mgr.pipeline_manager = pipeline.PipelineManager(
self.pipeline_cfg, self.transformer_manager)
polling_task = self.mgr.setup_polling_tasks().values()[0]
pollster = list(polling_task.pollster_matches[source_name])[0]
# 2 samples after 4 pollings, as pollster got disabled unpon exception
for x in range(0, 4):
self.mgr.interval_task(polling_task)
pub = self.mgr.pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(2, len(pub.samples))
LOG.error.assert_called_once_with(
('Prevent pollster %(name)s for '
'polling source %(source)s anymore!') % ({
'name': pollster.name,
'source': source_name
}))
def test_multiple_pipeline_exception(self):
self.pipeline_cfg.append({
'name': "second_pipeline",
"interval": 5,
'counters': ['b'],
'transformers': [{
'name': 'update',
'parameters':
{
"append_name": "_new",
}
}],
'publishers': ['except'],
})
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter])
self.test_counter = self.test_counter._replace(name='b')
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(len(publisher.counters), 1)
self.assertEqual(getattr(publisher.counters[0], "name"), 'a_update')
self.assertTrue(len(self.TransformerClass.samples) == 2)
self.assertTrue(getattr(self.TransformerClass.samples[0], "name")
== 'a')
self.assertTrue(getattr(self.TransformerClass.samples[1], "name")
== 'b')
def test_when_keystone_fail(self):
"""Test for bug 1316532."""
self.useFixture(
mockpatch.Patch('keystoneclient.v2_0.client.Client',
side_effect=Exception))
self.pipeline_cfg = {
'sources': [{
'name':
"test_keystone",
'interval':
10,
'meters': ['testkeystone'],
'resources': ['test://'] if self.source_resources else [],
'sinks': ['test_sink']
}],
'sinks': [{
'name': 'test_sink',
'transformers': [],
'publishers': ["test"]
}]
}
self.mgr.pipeline_manager = pipeline.PipelineManager(
self.pipeline_cfg, self.transformer_manager)
polling_tasks = self.mgr.setup_polling_tasks()
task = list(polling_tasks.values())[0]
self.mgr.interval_task(task)
self.assertFalse(self.PollsterKeystone.samples)
def test_interval_exception_isolation(self):
self.pipeline_cfg = [
{
'name': "test_pipeline_1",
'interval': 10,
'counters': ['testexceptionanother'],
'transformers': [],
'publishers': ["test_pub"],
},
{
'name': "test_pipeline_2",
'interval': 10,
'counters': ['testexception'],
'transformers': [],
'publishers': ["test_pub"],
},
]
self.mgr.pipeline_manager = pipeline.PipelineManager(
self.pipeline_cfg, self.transformer_manager,
self.publisher_manager)
polling_tasks = self.mgr.setup_polling_tasks()
self.assertEqual(len(polling_tasks.keys()), 1)
polling_tasks.get(10)
self.mgr.interval_task(polling_tasks.get(10))
self.assertEqual(len(self.publisher.counters), 0)
def test_rate_of_change_mapping(self):
map_from = {'name': 'disk\\.(read|write)\\.(bytes|requests)',
'unit': '(B|request)'}
map_to = {'name': 'disk.\\1.\\2.rate',
'unit': '\\1/s'}
self.pipeline_cfg[0]['transformers'] = [
{
'name': 'rate_of_change',
'parameters': {
'source': {
'map_from': map_from
},
'target': {
'map_to': map_to,
'type': sample.TYPE_GAUGE
},
},
},
]
self.pipeline_cfg[0]['counters'] = ['disk.read.bytes',
'disk.write.requests']
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
pipe = pipeline_manager.pipelines[0]
meters = ('disk.read.bytes', 'disk.write.requests')
units = ('B', 'request')
self._do_test_rate_of_change_mapping(pipe, meters, units)
def test_name(self):
pipeline_manager = pipeline.PipelineManager(self.CONF,
self.pipeline_cfg,
self.transformer_manager,
self.p_type)
for pipe in pipeline_manager.pipelines:
self.assertTrue(pipe.name.startswith('event:'))
def test_resources(self):
resources = ['test1://', 'test2://']
self.pipeline_cfg[0]['resources'] = resources
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
self.assertEqual(pipeline_manager.pipelines[0].resources,
resources)
def test_flush_pipeline_cache(self):
CACHE_SIZE = 10
extra_transformer_cfg = [
{
'name': 'cache',
'parameters': {
'size': CACHE_SIZE,
}
},
{
'name': 'update',
'parameters': {
'append_name': '_new'
}
},
]
self._extend_pipeline_cfg('transformers', extra_transformer_cfg)
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
pipe = pipeline_manager.pipelines[0]
pipe.publish_sample(None, self.test_counter)
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(0, len(publisher.samples))
pipe.flush(None)
self.assertEqual(0, len(publisher.samples))
pipe.publish_sample(None, self.test_counter)
pipe.flush(None)
self.assertEqual(0, len(publisher.samples))
for i in range(CACHE_SIZE - 2):
pipe.publish_sample(None, self.test_counter)
pipe.flush(None)
self.assertEqual(CACHE_SIZE, len(publisher.samples))
self.assertEqual('a_update_new', getattr(publisher.samples[0], 'name'))
def test_multiple_transformer_drop_transformer(self):
transformer_cfg = [
{
'name': 'update',
'parameters': {
"append_name": "_update",
}
},
{
'name': 'drop',
'parameters': {}
},
{
'name': 'update',
'parameters': {
"append_name": "_new",
}
},
]
self._set_pipeline_cfg('transformers', transformer_cfg)
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(0, len(publisher.samples))
self.assertEqual(1, len(self.TransformerClass.samples))
self.assertEqual('a', getattr(self.TransformerClass.samples[0],
'name'))
self.assertEqual(1, len(self.TransformerClassDrop.samples))
self.assertEqual('a_update',
getattr(self.TransformerClassDrop.samples[0], 'name'))
def test_wildcard_excluded_events(self):
event_cfg = ['*', '!a']
self._set_pipeline_cfg('events', event_cfg)
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.p_type)
self.assertFalse(pipeline_manager.pipelines[0].support_event('a'))
def test_wildcard_excluded_counters(self):
counter_cfg = ['*', '!a']
self.pipeline_cfg[0]['counters'] = counter_cfg
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.publisher_manager)
self.assertFalse(pipeline_manager.pipelines[0].support_counter('a'))
def test_flush_pipeline_cache(self):
CACHE_SIZE = 10
self.pipeline_cfg[0]['transformers'].extend([
{
'name': 'cache',
'parameters': {
'size': CACHE_SIZE,
}
},
{
'name': 'update',
'parameters': {
'append_name': '_new'
}
},
])
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
pipe = pipeline_manager.pipelines[0]
pipe.publish_sample(None, self.test_counter)
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(len(publisher.samples), 0)
pipe.flush(None)
self.assertEqual(len(publisher.samples), 0)
pipe.publish_sample(None, self.test_counter)
pipe.flush(None)
self.assertEqual(len(publisher.samples), 0)
for i in range(CACHE_SIZE - 2):
pipe.publish_sample(None, self.test_counter)
pipe.flush(None)
self.assertEqual(len(publisher.samples), CACHE_SIZE)
self.assertTrue(
getattr(publisher.samples[0], 'name') == 'a_update_new')
def test_multiple_transformer_same_class(self):
transformer_cfg = [
{
'name': 'update',
'parameters': {}
},
{
'name': 'update',
'parameters': {}
},
]
self._set_pipeline_cfg('transformers', transformer_cfg)
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(1, publisher.calls)
self.assertEqual(1, len(publisher.samples))
self.assertEqual('a_update_update',
getattr(publisher.samples[0], 'name'))
self.assertEqual(2, len(self.TransformerClass.samples))
self.assertEqual('a', getattr(self.TransformerClass.samples[0],
'name'))
self.assertEqual('a_update',
getattr(self.TransformerClass.samples[1], 'name'))
def test_get_interval(self):
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.publisher_manager)
pipe = pipeline_manager.pipelines[0]
self.assertTrue(pipe.get_interval() == 5)
def setup_pipeline(self):
self.transformer_manager = transformer.TransformerExtensionManager(
'ceilometer.transformer',
)
self.mgr.pipeline_manager = pipeline.PipelineManager(
self.pipeline_cfg,
self.transformer_manager)
def test_multiple_pipeline_exception(self):
self._break_pipeline_cfg()
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
self.test_counter = sample.Sample(
name='b',
type=self.test_counter.type,
volume=self.test_counter.volume,
unit=self.test_counter.unit,
user_id=self.test_counter.user_id,
project_id=self.test_counter.project_id,
resource_id=self.test_counter.resource_id,
timestamp=self.test_counter.timestamp,
resource_metadata=self.test_counter.resource_metadata,
)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(1, publisher.calls)
self.assertEqual(1, len(publisher.samples))
self.assertEqual('a_update', getattr(publisher.samples[0], "name"))
self.assertEqual(2, len(self.TransformerClass.samples))
self.assertEqual('a', getattr(self.TransformerClass.samples[0],
"name"))
self.assertEqual('b', getattr(self.TransformerClass.samples[1],
"name"))
def test_multiple_transformer_same_class(self):
self.pipeline_cfg[0]['transformers'] = [
{
'name': 'update',
'parameters': {}
},
{
'name': 'update',
'parameters': {}
},
]
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(publisher.calls, 1)
self.assertEqual(len(publisher.samples), 1)
self.assertEqual(getattr(publisher.samples[0], 'name'),
'a_update_update')
self.assertTrue(len(self.TransformerClass.samples) == 2)
self.assertTrue(
getattr(self.TransformerClass.samples[0], 'name') == 'a')
self.assertTrue(
getattr(self.TransformerClass.samples[1], 'name') == 'a_update')
def test_multiple_transformer_drop_transformer(self):
self.pipeline_cfg[0]['transformers'] = [
{
'name': 'update',
'parameters': {
"append_name": "_update",
}
},
{
'name': 'drop',
'parameters': {}
},
{
'name': 'update',
'parameters': {
"append_name": "_new",
}
},
]
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(len(publisher.samples), 0)
self.assertTrue(len(self.TransformerClass.samples) == 1)
self.assertTrue(
getattr(self.TransformerClass.samples[0], 'name') == 'a')
self.assertTrue(len(self.TransformerClassDrop.samples) == 1)
self.assertTrue(
getattr(self.TransformerClassDrop.samples[0], 'name') ==
'a_update')
def test_process_notification(self):
pub = self.PublisherClass()
transformer_manager = transformer.TransformerExtensionManager(
'ceilometer.transformer', )
publisher_manager = publisher.PublisherExtensionManager('fake', )
publisher_manager.extensions = [
extension.Extension(
'test_pub',
None,
None,
pub,
),
]
publisher_manager.by_name = dict(
(e.name, e) for e in publisher_manager.extensions)
notifier._pipeline_manager = pipeline.PipelineManager(
[{
'name': "test_pipeline",
'interval': 60,
'counters': ['*'],
'transformers': [],
'publishers': ["test_pub"],
}], transformer_manager, publisher_manager)
self.assertEqual(len(pub.counters), 0)
notifier.notify(None, MESSAGE)
self.assertTrue(len(pub.counters) > 0)
self.assertIn('disk.ephemeral.size', [c.name for c in pub.counters])
def test_multiple_included_counters(self):
counter_cfg = ['a', 'b']
self.pipeline_cfg[0]['counters'] = counter_cfg
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(len(publisher.samples), 1)
self.test_counter = sample.Sample(
name='b',
type=self.test_counter.type,
volume=self.test_counter.volume,
unit=self.test_counter.unit,
user_id=self.test_counter.user_id,
project_id=self.test_counter.project_id,
resource_id=self.test_counter.resource_id,
timestamp=self.test_counter.timestamp,
resource_metadata=self.test_counter.resource_metadata,
)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
self.assertEqual(len(publisher.samples), 2)
self.assertTrue(len(self.TransformerClass.samples) == 2)
self.assertEqual(getattr(publisher.samples[0], "name"), 'a_update')
self.assertEqual(getattr(publisher.samples[1], "name"), 'b_update')
def test_when_keystone_fail(self):
"""Test for bug 1316532."""
self.useFixture(mockpatch.Patch(
'keystoneclient.v2_0.client.Client',
side_effect=Exception))
self.pipeline_cfg = [
{
'name': "test_keystone",
'interval': 10,
'counters': ['testkeystone'],
'resources': ['test://'] if self.source_resources else [],
'transformers': [],
'publishers': ["test"],
},
]
self.mgr.pipeline_manager = pipeline.PipelineManager(
self.pipeline_cfg,
self.transformer_manager)
self.mgr.extensions = itertools.chain(
self.mgr.extensions,
[extension.Extension('testkeystone', None, None,
self.PollsterKeystone())])
polling_tasks = self.mgr.setup_polling_tasks()
self.mgr.interval_task(polling_tasks.values()[0])
self.assertFalse(self.PollsterKeystone.samples)
def test_interval_exception_isolation(self):
self.pipeline_cfg = [
{
'name': "test_pipeline_1",
'interval': 10,
'counters': ['testexceptionanother'],
'resources': ['test://'] if self.source_resources else [],
'transformers': [],
'publishers': ["test"],
},
{
'name': "test_pipeline_2",
'interval': 10,
'counters': ['testexception'],
'resources': ['test://'] if self.source_resources else [],
'transformers': [],
'publishers': ["test"],
},
]
self.mgr.pipeline_manager = pipeline.PipelineManager(
self.pipeline_cfg,
self.transformer_manager)
polling_tasks = self.mgr.setup_polling_tasks()
self.assertEqual(1, len(polling_tasks.keys()))
polling_tasks.get(10)
self.mgr.interval_task(polling_tasks.get(10))
pub = self.mgr.pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(0, len(pub.samples))
def test_variable_counter(self):
self.pipeline_cfg = [
{
'name': "test_pipeline",
'interval': 5,
'counters': ['a:*'],
'transformers': [{
'name': "update",
'parameters': {}
}],
'publishers': ["test"],
},
]
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.publisher_manager)
self.test_counter = self.test_counter._replace(name='a:b')
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter])
self.assertTrue(len(self.publisher.counters) == 1)
self.assertTrue(len(self.TransformerClass.samples) == 1)
self.assertTrue(
getattr(self.publisher.counters[0], "name") == 'a:b_update')
self.assertTrue(
getattr(self.TransformerClass.samples[0], "name") == 'a:b')
def test_flush_pipeline_cache_multiple_counter(self):
CACHE_SIZE = 3
self.pipeline_cfg[0]['transformers'].extend([
{
'name': 'cache',
'parameters': {
'size': CACHE_SIZE
}
},
{
'name': 'update',
'parameters': {
'append_name': '_new'
}
},
])
self.pipeline_cfg[0]['counters'] = ['a', 'b']
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.publisher_manager)
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter, self.test_counter._replace(name='b')])
self.assertTrue(len(self.publisher.counters) == 0)
self.assertEqual(len(self.publisher.counters), 0)
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter])
self.assertEqual(len(self.publisher.counters), CACHE_SIZE)
self.assertTrue(
getattr(self.publisher.counters[0], 'name') == 'a_update_new')
self.assertTrue(
getattr(self.publisher.counters[1], 'name') == 'b_update_new')
def test_multiple_transformer_same_class_different_parameter(self):
self.pipeline_cfg[0]['transformers'] = [
{
'name': 'update',
'parameters': {
"append_name": "_update",
}
},
{
'name': 'update',
'parameters': {
"append_name": "_new",
}
},
]
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.publisher_manager)
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter])
self.assertTrue(len(self.TransformerClass.samples) == 2)
self.assertTrue(
getattr(self.TransformerClass.samples[0], 'name') == 'a')
self.assertTrue(
getattr(self.TransformerClass.samples[1], 'name') == 'a_update')
self.assertTrue(len(self.publisher.counters) == 1)
self.assertTrue(
getattr(self.publisher.counters[0], 'name') == 'a_update_new')
def test_wildcard_and_excluded_wildcard_counters(self):
counter_cfg = ['*', '!disk.*']
self.pipeline_cfg[0]['counters'] = counter_cfg
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
self.assertFalse(
pipeline_manager.pipelines[0].support_meter('disk.read.bytes'))
self.assertTrue(pipeline_manager.pipelines[0].support_meter('cpu'))
def test_all_excluded_counters_is_excluded(self):
counter_cfg = ['!a', '!c']
self.pipeline_cfg[0]['counters'] = counter_cfg
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
self.assertFalse(pipeline_manager.pipelines[0].support_meter('a'))
self.assertTrue(pipeline_manager.pipelines[0].support_meter('b'))
self.assertFalse(pipeline_manager.pipelines[0].support_meter('c'))
def test_all_excluded_events_excluded(self):
event_cfg = ['!a', '!c']
self._set_pipeline_cfg('events', event_cfg)
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.p_type)
self.assertFalse(pipeline_manager.pipelines[0].support_event('a'))
self.assertTrue(pipeline_manager.pipelines[0].support_event('b'))
self.assertFalse(pipeline_manager.pipelines[0].support_event('c'))
def test_empty_transformer_pipeline(self):
self.pipeline_cfg[0]['transformers'] = []
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager)
with pipeline_manager.publisher(None) as p:
p([self.test_counter])
publisher = pipeline_manager.pipelines[0].publishers[0]
self.assertEqual(len(publisher.counters), 1)
self.assertEqual(getattr(publisher.counters[0], 'name'), 'a')
def test_empty_transformer_pipeline(self):
self.pipeline_cfg[0]['transformers'] = []
pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg,
self.transformer_manager,
self.publisher_manager)
with pipeline_manager.publisher(None, None) as p:
p([self.test_counter])
self.assertTrue(len(self.publisher.counters) == 1)
self.assertTrue(getattr(self.publisher.counters[0], 'name') == 'a')
