def test_polling_exception(self, LOG): source_name = 'test_pollingexception' self.pipeline_cfg = [ { 'name': source_name, 'interval': 10, 'counters': ['testpollingexception'], 'resources': ['test://'] if self.source_resources else [], 'transformers': [], 'publishers': ["test"], }, ] self.mgr.pipeline_manager = pipeline.PipelineManager( self.pipeline_cfg, self.transformer_manager) polling_task = self.mgr.setup_polling_tasks().values()[0] pollster = list(polling_task.pollster_matches[source_name])[0] # 2 samples after 4 pollings, as pollster got disabled unpon exception for x in range(0, 4): self.mgr.interval_task(polling_task) pub = self.mgr.pipeline_manager.pipelines[0].publishers[0] self.assertEqual(2, len(pub.samples)) LOG.error.assert_called_once_with( ('Prevent pollster %(name)s for ' 'polling source %(source)s anymore!') % ({ 'name': pollster.name, 'source': source_name }))
def test_multiple_pipeline_exception(self): self.pipeline_cfg.append({ 'name': "second_pipeline", "interval": 5, 'counters': ['b'], 'transformers': [{ 'name': 'update', 'parameters': { "append_name": "_new", } }], 'publishers': ['except'], }) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None, None) as p: p([self.test_counter]) self.test_counter = self.test_counter._replace(name='b') with pipeline_manager.publisher(None, None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(len(publisher.counters), 1) self.assertEqual(getattr(publisher.counters[0], "name"), 'a_update') self.assertTrue(len(self.TransformerClass.samples) == 2) self.assertTrue(getattr(self.TransformerClass.samples[0], "name") == 'a') self.assertTrue(getattr(self.TransformerClass.samples[1], "name") == 'b')
def test_when_keystone_fail(self): """Test for bug 1316532.""" self.useFixture( mockpatch.Patch('keystoneclient.v2_0.client.Client', side_effect=Exception)) self.pipeline_cfg = { 'sources': [{ 'name': "test_keystone", 'interval': 10, 'meters': ['testkeystone'], 'resources': ['test://'] if self.source_resources else [], 'sinks': ['test_sink'] }], 'sinks': [{ 'name': 'test_sink', 'transformers': [], 'publishers': ["test"] }] } self.mgr.pipeline_manager = pipeline.PipelineManager( self.pipeline_cfg, self.transformer_manager) polling_tasks = self.mgr.setup_polling_tasks() task = list(polling_tasks.values())[0] self.mgr.interval_task(task) self.assertFalse(self.PollsterKeystone.samples)
def test_interval_exception_isolation(self): self.pipeline_cfg = [ { 'name': "test_pipeline_1", 'interval': 10, 'counters': ['testexceptionanother'], 'transformers': [], 'publishers': ["test_pub"], }, { 'name': "test_pipeline_2", 'interval': 10, 'counters': ['testexception'], 'transformers': [], 'publishers': ["test_pub"], }, ] self.mgr.pipeline_manager = pipeline.PipelineManager( self.pipeline_cfg, self.transformer_manager, self.publisher_manager) polling_tasks = self.mgr.setup_polling_tasks() self.assertEqual(len(polling_tasks.keys()), 1) polling_tasks.get(10) self.mgr.interval_task(polling_tasks.get(10)) self.assertEqual(len(self.publisher.counters), 0)
def test_rate_of_change_mapping(self): map_from = {'name': 'disk\\.(read|write)\\.(bytes|requests)', 'unit': '(B|request)'} map_to = {'name': 'disk.\\1.\\2.rate', 'unit': '\\1/s'} self.pipeline_cfg[0]['transformers'] = [ { 'name': 'rate_of_change', 'parameters': { 'source': { 'map_from': map_from }, 'target': { 'map_to': map_to, 'type': sample.TYPE_GAUGE }, }, }, ] self.pipeline_cfg[0]['counters'] = ['disk.read.bytes', 'disk.write.requests'] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) pipe = pipeline_manager.pipelines[0] meters = ('disk.read.bytes', 'disk.write.requests') units = ('B', 'request') self._do_test_rate_of_change_mapping(pipe, meters, units)
def test_name(self): pipeline_manager = pipeline.PipelineManager(self.CONF, self.pipeline_cfg, self.transformer_manager, self.p_type) for pipe in pipeline_manager.pipelines: self.assertTrue(pipe.name.startswith('event:'))
def test_resources(self): resources = ['test1://', 'test2://'] self.pipeline_cfg[0]['resources'] = resources pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) self.assertEqual(pipeline_manager.pipelines[0].resources, resources)
def test_flush_pipeline_cache(self): CACHE_SIZE = 10 extra_transformer_cfg = [ { 'name': 'cache', 'parameters': { 'size': CACHE_SIZE, } }, { 'name': 'update', 'parameters': { 'append_name': '_new' } }, ] self._extend_pipeline_cfg('transformers', extra_transformer_cfg) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) pipe = pipeline_manager.pipelines[0] pipe.publish_sample(None, self.test_counter) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(0, len(publisher.samples)) pipe.flush(None) self.assertEqual(0, len(publisher.samples)) pipe.publish_sample(None, self.test_counter) pipe.flush(None) self.assertEqual(0, len(publisher.samples)) for i in range(CACHE_SIZE - 2): pipe.publish_sample(None, self.test_counter) pipe.flush(None) self.assertEqual(CACHE_SIZE, len(publisher.samples)) self.assertEqual('a_update_new', getattr(publisher.samples[0], 'name'))
def test_multiple_transformer_drop_transformer(self): transformer_cfg = [ { 'name': 'update', 'parameters': { "append_name": "_update", } }, { 'name': 'drop', 'parameters': {} }, { 'name': 'update', 'parameters': { "append_name": "_new", } }, ] self._set_pipeline_cfg('transformers', transformer_cfg) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(0, len(publisher.samples)) self.assertEqual(1, len(self.TransformerClass.samples)) self.assertEqual('a', getattr(self.TransformerClass.samples[0], 'name')) self.assertEqual(1, len(self.TransformerClassDrop.samples)) self.assertEqual('a_update', getattr(self.TransformerClassDrop.samples[0], 'name'))
def test_wildcard_excluded_events(self): event_cfg = ['*', '!a'] self._set_pipeline_cfg('events', event_cfg) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.p_type) self.assertFalse(pipeline_manager.pipelines[0].support_event('a'))
def test_wildcard_excluded_counters(self): counter_cfg = ['*', '!a'] self.pipeline_cfg[0]['counters'] = counter_cfg pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.publisher_manager) self.assertFalse(pipeline_manager.pipelines[0].support_counter('a'))
def test_flush_pipeline_cache(self): CACHE_SIZE = 10 self.pipeline_cfg[0]['transformers'].extend([ { 'name': 'cache', 'parameters': { 'size': CACHE_SIZE, } }, { 'name': 'update', 'parameters': { 'append_name': '_new' } }, ]) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) pipe = pipeline_manager.pipelines[0] pipe.publish_sample(None, self.test_counter) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(len(publisher.samples), 0) pipe.flush(None) self.assertEqual(len(publisher.samples), 0) pipe.publish_sample(None, self.test_counter) pipe.flush(None) self.assertEqual(len(publisher.samples), 0) for i in range(CACHE_SIZE - 2): pipe.publish_sample(None, self.test_counter) pipe.flush(None) self.assertEqual(len(publisher.samples), CACHE_SIZE) self.assertTrue( getattr(publisher.samples[0], 'name') == 'a_update_new')
def test_multiple_transformer_same_class(self): transformer_cfg = [ { 'name': 'update', 'parameters': {} }, { 'name': 'update', 'parameters': {} }, ] self._set_pipeline_cfg('transformers', transformer_cfg) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(1, publisher.calls) self.assertEqual(1, len(publisher.samples)) self.assertEqual('a_update_update', getattr(publisher.samples[0], 'name')) self.assertEqual(2, len(self.TransformerClass.samples)) self.assertEqual('a', getattr(self.TransformerClass.samples[0], 'name')) self.assertEqual('a_update', getattr(self.TransformerClass.samples[1], 'name'))
def test_get_interval(self): pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.publisher_manager) pipe = pipeline_manager.pipelines[0] self.assertTrue(pipe.get_interval() == 5)
def setup_pipeline(self): self.transformer_manager = transformer.TransformerExtensionManager( 'ceilometer.transformer', ) self.mgr.pipeline_manager = pipeline.PipelineManager( self.pipeline_cfg, self.transformer_manager)
def test_multiple_pipeline_exception(self): self._break_pipeline_cfg() pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) self.test_counter = sample.Sample( name='b', type=self.test_counter.type, volume=self.test_counter.volume, unit=self.test_counter.unit, user_id=self.test_counter.user_id, project_id=self.test_counter.project_id, resource_id=self.test_counter.resource_id, timestamp=self.test_counter.timestamp, resource_metadata=self.test_counter.resource_metadata, ) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(1, publisher.calls) self.assertEqual(1, len(publisher.samples)) self.assertEqual('a_update', getattr(publisher.samples[0], "name")) self.assertEqual(2, len(self.TransformerClass.samples)) self.assertEqual('a', getattr(self.TransformerClass.samples[0], "name")) self.assertEqual('b', getattr(self.TransformerClass.samples[1], "name"))
def test_multiple_transformer_same_class(self): self.pipeline_cfg[0]['transformers'] = [ { 'name': 'update', 'parameters': {} }, { 'name': 'update', 'parameters': {} }, ] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(publisher.calls, 1) self.assertEqual(len(publisher.samples), 1) self.assertEqual(getattr(publisher.samples[0], 'name'), 'a_update_update') self.assertTrue(len(self.TransformerClass.samples) == 2) self.assertTrue( getattr(self.TransformerClass.samples[0], 'name') == 'a') self.assertTrue( getattr(self.TransformerClass.samples[1], 'name') == 'a_update')
def test_multiple_transformer_drop_transformer(self): self.pipeline_cfg[0]['transformers'] = [ { 'name': 'update', 'parameters': { "append_name": "_update", } }, { 'name': 'drop', 'parameters': {} }, { 'name': 'update', 'parameters': { "append_name": "_new", } }, ] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(len(publisher.samples), 0) self.assertTrue(len(self.TransformerClass.samples) == 1) self.assertTrue( getattr(self.TransformerClass.samples[0], 'name') == 'a') self.assertTrue(len(self.TransformerClassDrop.samples) == 1) self.assertTrue( getattr(self.TransformerClassDrop.samples[0], 'name') == 'a_update')
def test_process_notification(self): pub = self.PublisherClass() transformer_manager = transformer.TransformerExtensionManager( 'ceilometer.transformer', ) publisher_manager = publisher.PublisherExtensionManager('fake', ) publisher_manager.extensions = [ extension.Extension( 'test_pub', None, None, pub, ), ] publisher_manager.by_name = dict( (e.name, e) for e in publisher_manager.extensions) notifier._pipeline_manager = pipeline.PipelineManager( [{ 'name': "test_pipeline", 'interval': 60, 'counters': ['*'], 'transformers': [], 'publishers': ["test_pub"], }], transformer_manager, publisher_manager) self.assertEqual(len(pub.counters), 0) notifier.notify(None, MESSAGE) self.assertTrue(len(pub.counters) > 0) self.assertIn('disk.ephemeral.size', [c.name for c in pub.counters])
def test_multiple_included_counters(self): counter_cfg = ['a', 'b'] self.pipeline_cfg[0]['counters'] = counter_cfg pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(len(publisher.samples), 1) self.test_counter = sample.Sample( name='b', type=self.test_counter.type, volume=self.test_counter.volume, unit=self.test_counter.unit, user_id=self.test_counter.user_id, project_id=self.test_counter.project_id, resource_id=self.test_counter.resource_id, timestamp=self.test_counter.timestamp, resource_metadata=self.test_counter.resource_metadata, ) with pipeline_manager.publisher(None) as p: p([self.test_counter]) self.assertEqual(len(publisher.samples), 2) self.assertTrue(len(self.TransformerClass.samples) == 2) self.assertEqual(getattr(publisher.samples[0], "name"), 'a_update') self.assertEqual(getattr(publisher.samples[1], "name"), 'b_update')
def test_when_keystone_fail(self): """Test for bug 1316532.""" self.useFixture(mockpatch.Patch( 'keystoneclient.v2_0.client.Client', side_effect=Exception)) self.pipeline_cfg = [ { 'name': "test_keystone", 'interval': 10, 'counters': ['testkeystone'], 'resources': ['test://'] if self.source_resources else [], 'transformers': [], 'publishers': ["test"], }, ] self.mgr.pipeline_manager = pipeline.PipelineManager( self.pipeline_cfg, self.transformer_manager) self.mgr.extensions = itertools.chain( self.mgr.extensions, [extension.Extension('testkeystone', None, None, self.PollsterKeystone())]) polling_tasks = self.mgr.setup_polling_tasks() self.mgr.interval_task(polling_tasks.values()[0]) self.assertFalse(self.PollsterKeystone.samples)
def test_interval_exception_isolation(self): self.pipeline_cfg = [ { 'name': "test_pipeline_1", 'interval': 10, 'counters': ['testexceptionanother'], 'resources': ['test://'] if self.source_resources else [], 'transformers': [], 'publishers': ["test"], }, { 'name': "test_pipeline_2", 'interval': 10, 'counters': ['testexception'], 'resources': ['test://'] if self.source_resources else [], 'transformers': [], 'publishers': ["test"], }, ] self.mgr.pipeline_manager = pipeline.PipelineManager( self.pipeline_cfg, self.transformer_manager) polling_tasks = self.mgr.setup_polling_tasks() self.assertEqual(1, len(polling_tasks.keys())) polling_tasks.get(10) self.mgr.interval_task(polling_tasks.get(10)) pub = self.mgr.pipeline_manager.pipelines[0].publishers[0] self.assertEqual(0, len(pub.samples))
def test_variable_counter(self): self.pipeline_cfg = [ { 'name': "test_pipeline", 'interval': 5, 'counters': ['a:*'], 'transformers': [{ 'name': "update", 'parameters': {} }], 'publishers': ["test"], }, ] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.publisher_manager) self.test_counter = self.test_counter._replace(name='a:b') with pipeline_manager.publisher(None, None) as p: p([self.test_counter]) self.assertTrue(len(self.publisher.counters) == 1) self.assertTrue(len(self.TransformerClass.samples) == 1) self.assertTrue( getattr(self.publisher.counters[0], "name") == 'a:b_update') self.assertTrue( getattr(self.TransformerClass.samples[0], "name") == 'a:b')
def test_flush_pipeline_cache_multiple_counter(self): CACHE_SIZE = 3 self.pipeline_cfg[0]['transformers'].extend([ { 'name': 'cache', 'parameters': { 'size': CACHE_SIZE } }, { 'name': 'update', 'parameters': { 'append_name': '_new' } }, ]) self.pipeline_cfg[0]['counters'] = ['a', 'b'] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.publisher_manager) with pipeline_manager.publisher(None, None) as p: p([self.test_counter, self.test_counter._replace(name='b')]) self.assertTrue(len(self.publisher.counters) == 0) self.assertEqual(len(self.publisher.counters), 0) with pipeline_manager.publisher(None, None) as p: p([self.test_counter]) self.assertEqual(len(self.publisher.counters), CACHE_SIZE) self.assertTrue( getattr(self.publisher.counters[0], 'name') == 'a_update_new') self.assertTrue( getattr(self.publisher.counters[1], 'name') == 'b_update_new')
def test_multiple_transformer_same_class_different_parameter(self): self.pipeline_cfg[0]['transformers'] = [ { 'name': 'update', 'parameters': { "append_name": "_update", } }, { 'name': 'update', 'parameters': { "append_name": "_new", } }, ] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.publisher_manager) with pipeline_manager.publisher(None, None) as p: p([self.test_counter]) self.assertTrue(len(self.TransformerClass.samples) == 2) self.assertTrue( getattr(self.TransformerClass.samples[0], 'name') == 'a') self.assertTrue( getattr(self.TransformerClass.samples[1], 'name') == 'a_update') self.assertTrue(len(self.publisher.counters) == 1) self.assertTrue( getattr(self.publisher.counters[0], 'name') == 'a_update_new')
def test_wildcard_and_excluded_wildcard_counters(self): counter_cfg = ['*', '!disk.*'] self.pipeline_cfg[0]['counters'] = counter_cfg pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) self.assertFalse( pipeline_manager.pipelines[0].support_meter('disk.read.bytes')) self.assertTrue(pipeline_manager.pipelines[0].support_meter('cpu'))
def test_all_excluded_counters_is_excluded(self): counter_cfg = ['!a', '!c'] self.pipeline_cfg[0]['counters'] = counter_cfg pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) self.assertFalse(pipeline_manager.pipelines[0].support_meter('a')) self.assertTrue(pipeline_manager.pipelines[0].support_meter('b')) self.assertFalse(pipeline_manager.pipelines[0].support_meter('c'))
def test_all_excluded_events_excluded(self): event_cfg = ['!a', '!c'] self._set_pipeline_cfg('events', event_cfg) pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.p_type) self.assertFalse(pipeline_manager.pipelines[0].support_event('a')) self.assertTrue(pipeline_manager.pipelines[0].support_event('b')) self.assertFalse(pipeline_manager.pipelines[0].support_event('c'))
def test_empty_transformer_pipeline(self): self.pipeline_cfg[0]['transformers'] = [] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager) with pipeline_manager.publisher(None) as p: p([self.test_counter]) publisher = pipeline_manager.pipelines[0].publishers[0] self.assertEqual(len(publisher.counters), 1) self.assertEqual(getattr(publisher.counters[0], 'name'), 'a')
def test_empty_transformer_pipeline(self): self.pipeline_cfg[0]['transformers'] = [] pipeline_manager = pipeline.PipelineManager(self.pipeline_cfg, self.transformer_manager, self.publisher_manager) with pipeline_manager.publisher(None, None) as p: p([self.test_counter]) self.assertTrue(len(self.publisher.counters) == 1) self.assertTrue(getattr(self.publisher.counters[0], 'name') == 'a')