def test_broken_config_load(self, mylog): contents = [ ("---\n" "resources:\n" " - resource_type: foobar\n"), ("---\n" "resources:\n" " - resource_type: 0\n"), ("---\n" "resources:\n" " - sample_types: ['foo', 'bar']\n"), ("---\n" "resources:\n" " - sample_types: foobar\n" " - resource_type: foobar\n"), ] for content in contents: if six.PY3: content = content.encode('utf-8') temp = fileutils.write_to_tempfile(content=content, prefix='gnocchi_resources', suffix='.yaml') self.addCleanup(os.remove, temp) self.conf.config(filter_service_activity=False, resources_definition_file=temp, group='dispatcher_gnocchi') d = gnocchi.GnocchiDispatcher(self.conf.conf) self.assertTrue(mylog.error.called) self.assertEqual(0, len(d.resources_definition))
def test_config_load(self): self.conf.config(filter_service_activity=False, group='dispatcher_gnocchi') d = gnocchi.GnocchiDispatcher(self.conf.conf) names = [rd.cfg['resource_type'] for rd in d.resources_definition] self.assertIn('instance', names) self.assertIn('volume', names)
def test_event_workflow(self, fakeclient_cls): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value fakeclient.resource.search.side_effect = [ [{ "id": "b26268d6-8bb5-11e6-baff-00224d8226cd", "type": "instance_disk", "instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1" }], [{ "id": "b1c7544a-8bb5-11e6-850e-00224d8226cd", "type": "instance_network_interface", "instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1" }], ] search_params = { '=': { 'instance_id': '9f9d01b9-4a58-4271-9e27-398b21ab20d1' } } now = timeutils.utcnow() self.useFixture(utils_fixture.TimeFixture(now)) expected_calls = [ mock.call.capabilities.list(), mock.call.resource.search('instance_disk', search_params), mock.call.resource.search('instance_network_interface', search_params), mock.call.resource.update('instance', '9f9d01b9-4a58-4271-9e27-398b21ab20d1', {'ended_at': now.isoformat()}), mock.call.resource.update('instance_disk', 'b26268d6-8bb5-11e6-baff-00224d8226cd', {'ended_at': now.isoformat()}), mock.call.resource.update('instance_network_interface', 'b1c7544a-8bb5-11e6-850e-00224d8226cd', {'ended_at': now.isoformat()}), mock.call.resource.update('image', 'dc337359-de70-4044-8e2c-80573ba6e577', {'ended_at': now.isoformat()}), mock.call.resource.update('volume', '6cc6e7dd-d17d-460f-ae79-7e08a216ce96', {'ended_at': now.isoformat()}), mock.call.resource.update('network', '705e2c08-08e8-45cb-8673-5c5be955569b', {'ended_at': now.isoformat()}) ] self.dispatcher.record_events([ INSTANCE_DELETE_START, IMAGE_DELETE_START, VOLUME_DELETE_START, FLOATINGIP_DELETE_END ]) self.assertEqual(9, len(fakeclient.mock_calls)) for call in expected_calls: self.assertIn(call, fakeclient.mock_calls)
def _do_test_activity_filter(self, expected_samples, fake_process_samples): d = gnocchi.GnocchiDispatcher(self.conf.conf) d.record_metering_data(self.samples) fake_process_samples.assert_called_with( mock.ANY, self.resource_id, 'disk.root.size', expected_samples, True, )
def _do_test_activity_filter(self, expected_measures, fake_batch, __): d = gnocchi.GnocchiDispatcher(self.conf.conf) d.record_metering_data(self.samples) fake_batch.assert_called_with(mock.ANY, mock.ANY, { 'metrics': 1, 'resources': 1, 'measures': expected_measures })
def setUp(self): super(DispatcherWorkflowTest, self).setUp() self.conf = self.useFixture(config_fixture.Config()) ks_client = mock.Mock(auth_token='fake_token') ks_client.tenants.find.return_value = mock.Mock( name='gnocchi', id='a2d42c23-d518-46b6-96ab-3fba2e146859') self.useFixture(mockpatch.Patch( 'ceilometer.keystone_client.get_client', return_value=ks_client)) ceilometer_service.prepare_service([]) self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) self.sample['resource_id'] = str(uuid.uuid4())
def test_archive_policy_map_config(self): archive_policy_map = yaml.dump({'foo.*': 'low'}) archive_policy_cfg_file = tempfile.NamedTemporaryFile(mode='w+b', prefix="foo", suffix=".yaml") archive_policy_cfg_file.write(archive_policy_map.encode()) archive_policy_cfg_file.seek(0) self.conf.conf.dispatcher_gnocchi.archive_policy_file = ( archive_policy_cfg_file.name) d = gnocchi.GnocchiDispatcher(self.conf.conf) legacy = d._load_archive_policy(self.conf.conf) self.assertEqual(legacy.get('foo.disk.rate'), "low") archive_policy_cfg_file.close()
def _do_test_activity_filter(self, expected_samples, fake_process_resource): def assert_samples(resource_id, metric_grouped_samples): samples = [] for metric_name, s in metric_grouped_samples: samples.extend(list(s)) self.assertEqual(expected_samples, samples) fake_process_resource.side_effect = assert_samples d = gnocchi.GnocchiDispatcher(self.conf.conf) d.record_metering_data(self.samples) fake_process_resource.assert_called_with(self.resource_id, mock.ANY)
def test_unhandled_meter(self, fake_batch): samples = [{ 'counter_name': 'unknown.meter', 'counter_unit': 'GB', 'counter_type': 'gauge', 'counter_volume': '2', 'user_id': 'test_user', 'project_id': 'test_project', 'source': 'openstack', 'timestamp': '2014-05-08 20:23:48.028195', 'resource_id': 'randomid', 'resource_metadata': {} }] d = gnocchi.GnocchiDispatcher(self.conf.conf) d.record_metering_data(samples) self.assertEqual(0, len(fake_batch.call_args[0][1]))
def test_event_workflow(self, fakeclient_cls): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value fakeclient.resource.search.side_effect = [ [{ "id": "b26268d6-8bb5-11e6-baff-00224d8226cd", "type": "instance_disk", "instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1" }], [{ "id": "b1c7544a-8bb5-11e6-850e-00224d8226cd", "type": "instance_network_interface", "instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1" }], ] search_params = { '=': { 'instance_id': '9f9d01b9-4a58-4271-9e27-398b21ab20d1' } } now = timeutils.utcnow() self.useFixture(utils_fixture.TimeFixture(now)) expected_calls = [ mock.call.capabilities.list(), mock.call.resource.search('instance_disk', search_params), mock.call.resource.search('instance_network_interface', search_params), mock.call.resource.update('instance', '9f9d01b9-4a58-4271-9e27-398b21ab20d1', {'ended_at': now.isoformat()}), mock.call.resource.update('instance_disk', 'b26268d6-8bb5-11e6-baff-00224d8226cd', {'ended_at': now.isoformat()}), mock.call.resource.update('instance_network_interface', 'b1c7544a-8bb5-11e6-850e-00224d8226cd', {'ended_at': now.isoformat()}) ] self.dispatcher.record_events([INSTANCE_DELETE_START]) self.assertEqual(6, len(fakeclient.mock_calls)) for call in expected_calls: self.assertIn(call, fakeclient.mock_calls)
def setUp(self): super(DispatcherWorkflowTest, self).setUp() self.conf = self.useFixture(config_fixture.Config()) # Set this explicitly to avoid conflicts with any existing # configuration. self.conf.config(url='http://localhost:8041', group='dispatcher_gnocchi') ks_client = mock.Mock(auth_token='fake_token') ks_client.tenants.find.return_value = mock.Mock( name='gnocchi', id='a2d42c23-d518-46b6-96ab-3fba2e146859') self.useFixture( mockpatch.Patch('ceilometer.keystone_client.get_client', return_value=ks_client)) ceilometer_service.prepare_service([]) self.conf.config(resources_definition_file=self.path_get( 'etc/ceilometer/gnocchi_resources.yaml'), group="dispatcher_gnocchi") self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) self.sample['resource_id'] = str(uuid.uuid4())
def test_workflow(self, fakeclient_cls, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value resource_id = self.sample['resource_id'].replace("/", "_") metric_name = self.sample['counter_name'] gnocchi_id = uuid.uuid4() expected_calls = [ mock.call.metric.batch_resources_metrics_measures( {resource_id: { metric_name: self.measures_attributes }}, create_metrics=True) ] expected_debug = [ mock.call('gnocchi project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), ] measures_posted = False batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] elif not self.resource_exists: batch_side_effect += [ gnocchi_exc.BadRequest( 400, { "cause": "Unknown resources", 'detail': [{ 'resource_id': gnocchi_id, 'original_resource_id': resource_id }] }) ] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) for k, v in six.iteritems(attributes['metrics']): if k == 'disk.root.size': v['unit'] = 'GB' continue if k == 'hardware.ipmi.node.power': v['unit'] = 'W' continue expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] elif self.create_resource_race: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] else: # not resource_exists expected_debug.append( mock.call('Resource %s created', self.sample['resource_id'])) if not self.create_resource_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( {resource_id: { metric_name: self.measures_attributes }}, create_metrics=True)) if self.retry_post_measures_fail: batch_side_effect += [Exception('boom!')] else: measures_posted = True else: measures_posted = True if measures_posted: batch_side_effect += [None] expected_debug.append( mock.call( "%(measures)d measures posted against %(metrics)d " "metrics through %(resources)d resources", dict(measures=len(self.measures_attributes), metrics=1, resources=1))) if self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.update_resource_fail: fakeclient.resource.update.side_effect = [Exception('boom!')] else: expected_debug.append( mock.call('Resource %s updated', self.sample['resource_id'])) batch = fakeclient.metric.batch_resources_metrics_measures batch.side_effect = batch_side_effect self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if (self.post_measure_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) else: self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls) self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fake_requests, client_logger, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) base_url = self.dispatcher.conf.dispatcher_gnocchi.url url_params = { 'url': urlparse.urljoin(base_url, '/v1/resource'), 'resource_id': self.sample['resource_id'], 'resource_type': self.resource_type, 'metric_name': self.sample['counter_name'] } headers = { 'Content-Type': 'application/json', 'X-Auth-Token': 'fake_token' } patch_responses = [] post_responses = [] # This is needed to mock Exception in py3 fake_requests.ConnectionError = requests.ConnectionError expected_calls = [ mock.call.session(), mock.call.adapters.HTTPAdapter(pool_block=True), mock.call.session().mount('http://', mock.ANY), mock.call.session().mount('https://', mock.ANY), mock.call.session().post( "%(url)s/%(resource_type)s/%(resource_id)s/" "metric/%(metric_name)s/measures" % url_params, headers=headers, data=json_matcher(self.measures_attributes)) ] post_responses.append(MockResponse(self.measure)) if self.measure == 401: type(self.ks_client).auth_token = mock.PropertyMock(side_effect=[ 'fake_token', 'new_token', 'new_token', 'new_token', 'new_token' ]) headers = { 'Content-Type': 'application/json', 'X-Auth-Token': 'new_token' } expected_calls.append(mock.call.session().post( "%(url)s/%(resource_type)s/%(resource_id)s/" "metric/%(metric_name)s/measures" % url_params, headers=headers, data=json_matcher(self.measures_attributes))) post_responses.append(MockResponse(200)) if self.post_resource: attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict((metric_name, { 'archive_policy_name': 'low' }) for metric_name in self.metric_names) expected_calls.append( mock.call.session().post("%(url)s/%(resource_type)s" % url_params, headers=headers, data=json_matcher(attributes)), ) post_responses.append(MockResponse(self.post_resource)) if self.metric: expected_calls.append(mock.call.session().post( "%(url)s/%(resource_type)s/%(resource_id)s/metric" % url_params, headers=headers, data=json_matcher({ self.sample['counter_name']: { 'archive_policy_name': 'low' } }))) post_responses.append(MockResponse(self.metric)) if self.measure_retry: expected_calls.append(mock.call.session().post( "%(url)s/%(resource_type)s/%(resource_id)s/" "metric/%(metric_name)s/measures" % url_params, headers=headers, data=json_matcher(self.measures_attributes))) post_responses.append(MockResponse(self.measure_retry)) if self.patch_resource and self.patchable_attributes: expected_calls.append( mock.call.session().patch( "%(url)s/%(resource_type)s/%(resource_id)s" % url_params, headers=headers, data=json_matcher(self.patchable_attributes)), ) patch_responses.append(MockResponse(self.patch_resource)) s = fake_requests.session.return_value s.patch.side_effect = patch_responses s.post.side_effect = post_responses self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if self.measure == 500 or self.measure_retry == 500: logger.error.assert_called_with( "Fail to post measure on metric %s of resource %s " "with status: %d: Internal Server Error" % (self.sample['counter_name'], self.sample['resource_id'], 500)) elif self.post_resource == 500 or (self.patch_resource == 500 and self.patchable_attributes): logger.error.assert_called_with( "Resource %s %s failed with status: " "%d: Internal Server Error" % (self.sample['resource_id'], 'update' if self.patch_resource else 'creation', 500)) elif self.metric == 500: logger.error.assert_called_with( "Fail to create metric %s of resource %s " "with status: %d: Internal Server Error" % (self.sample['counter_name'], self.sample['resource_id'], 500)) elif self.patch_resource == 204 and self.patchable_attributes: client_logger.debug.assert_called_with('Resource %s updated', self.sample['resource_id']) elif self.measure == 200: client_logger.debug.assert_called_with( "Measure posted on metric %s of resource %s", self.sample['counter_name'], self.sample['resource_id']) self.assertEqual(expected_calls, fake_requests.mock_calls)
def test_workflow(self, fakeclient_cls, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value # FIXME(sileht): we don't use urlparse.quote here # to ensure / is converted in %2F # temporary disabled until we find a solution # on gnocchi side. Current gnocchiclient doesn't # encode the resource_id resource_id = self.sample['resource_id'] # .replace("/", "%2F"), metric_name = self.sample['counter_name'] gnocchi_id = gnocchi_utils.encode_resource_id(resource_id) expected_calls = [ mock.call.capabilities.list(), mock.call.metric.batch_resources_metrics_measures( {gnocchi_id: { metric_name: self.measures_attributes }}) ] expected_debug = [ mock.call('gnocchi project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), ] measures_posted = False batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] elif not self.resource_exists or not self.metric_exists: batch_side_effect += [ gnocchi_exc.BadRequest( 400, "Unknown metrics: %s/%s" % (gnocchi_id, metric_name)) ] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] elif self.resource_exists: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] expected_calls.append( mock.call.metric.create({ 'name': self.sample['counter_name'], 'resource_id': resource_id })) if self.create_metric_fail: fakeclient.metric.create.side_effect = [Exception('boom!')] elif self.metric_exists: fakeclient.metric.create.side_effect = [ gnocchi_exc.NamedMetricAreadyExists(409) ] else: fakeclient.metric.create.side_effect = [None] else: # not resource_exists expected_debug.append( mock.call('Resource %s created', self.sample['resource_id'])) if not self.create_resource_fail and not self.create_metric_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( {gnocchi_id: { metric_name: self.measures_attributes }})) if self.retry_post_measures_fail: batch_side_effect += [Exception('boom!')] else: measures_posted = True else: measures_posted = True if measures_posted: batch_side_effect += [None] expected_debug.append( mock.call( "%(measures)d measures posted against %(metrics)d " "metrics through %(resources)d resources", dict(measures=len(self.measures_attributes), metrics=1, resources=1))) if self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.update_resource_fail: fakeclient.resource.update.side_effect = [Exception('boom!')] else: expected_debug.append( mock.call('Resource %s updated', self.sample['resource_id'])) batch = fakeclient.metric.batch_resources_metrics_measures batch.side_effect = batch_side_effect self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if (self.post_measure_fail or self.create_metric_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) else: self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls) self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_archive_policy_default(self): d = gnocchi.GnocchiDispatcher(self.conf.conf) self.assertEqual(d.gnocchi_archive_policy_default, "low")
def test_workflow(self, fakeclient_cls, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value # FIXME(sileht): we don't use urlparse.quote here # to ensure / is converted in %2F # temporary disabled until we find a solution # on gnocchi side. Current gnocchiclient doesn't # encode the resource_id resource_id = self.sample['resource_id'] # .replace("/", "%2F"), metric_name = self.sample['counter_name'] expected_calls = [ mock.call.capabilities.list(), mock.call.metric.add_measures(metric_name, self.measures_attributes, resource_id) ] add_measures_side_effect = [] if self.measure == 404 and self.post_resource: add_measures_side_effect += [gnocchi_exc.ResourceNotFound(404)] elif self.measure == 404 and self.metric: add_measures_side_effect += [gnocchi_exc.MetricNotFound(404)] elif self.measure == 500: add_measures_side_effect += [Exception('boom!')] if self.post_resource: attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.post_resource == 409: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] elif self.post_resource == 500: fakeclient.resource.create.side_effect = [Exception('boom!')] if self.metric: expected_calls.append( mock.call.metric.create({ 'name': self.sample['counter_name'], 'resource_id': resource_id })) if self.metric == 409: fakeclient.metric.create.side_effect = [ gnocchi_exc.NamedMetricAreadyExists(409) ] elif self.metric == 500: fakeclient.metric.create.side_effect = [Exception('boom!')] if self.measure_retry: expected_calls.append( mock.call.metric.add_measures(metric_name, self.measures_attributes, resource_id)) if self.measure_retry == 204: add_measures_side_effect += [None] elif self.measure_retry == 500: add_measures_side_effect += [Exception('boom!')] else: add_measures_side_effect += [None] if self.patch_resource and self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.patch_resource == 500: fakeclient.resource.update.side_effect = [Exception('boom!')] fakeclient.metric.add_measures.side_effect = add_measures_side_effect self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if (self.measure == 500 or self.measure_retry == 500 or self.metric == 500 or self.post_resource == 500 or (self.patch_resource == 500 and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) elif self.patch_resource == 204 and self.patchable_attributes: logger.debug.assert_called_with('Resource %s updated', self.sample['resource_id']) self.assertEqual(0, logger.error.call_count) elif self.measure == 200: logger.debug.assert_called_with( "Measure posted on metric %s of resource %s", self.sample['counter_name'], self.sample['resource_id']) self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls)
def test_extensions_load(self): self.conf.config(filter_service_activity=False, group='dispatcher_gnocchi') d = gnocchi.GnocchiDispatcher(self.conf.conf) self.assertIn('instance', d.mgmr.names()) self.assertIn('volume', d.mgmr.names())