def test_broken_config_load(self, mylog):
contents = [
("---\n"
"resources:\n"
" - resource_type: foobar\n"),
("---\n"
"resources:\n"
" - resource_type: 0\n"),
("---\n"
"resources:\n"
" - sample_types: ['foo', 'bar']\n"),
("---\n"
"resources:\n"
" - sample_types: foobar\n"
" - resource_type: foobar\n"),
]
for content in contents:
if six.PY3:
content = content.encode('utf-8')
temp = fileutils.write_to_tempfile(content=content,
prefix='gnocchi_resources',
suffix='.yaml')
self.addCleanup(os.remove, temp)
self.conf.config(filter_service_activity=False,
resources_definition_file=temp,
group='dispatcher_gnocchi')
d = gnocchi.GnocchiDispatcher(self.conf.conf)
self.assertTrue(mylog.error.called)
self.assertEqual(0, len(d.resources_definition))
def test_config_load(self):
self.conf.config(filter_service_activity=False,
group='dispatcher_gnocchi')
d = gnocchi.GnocchiDispatcher(self.conf.conf)
names = [rd.cfg['resource_type'] for rd in d.resources_definition]
self.assertIn('instance', names)
self.assertIn('volume', names)
def test_event_workflow(self, fakeclient_cls):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
fakeclient.resource.search.side_effect = [
[{
"id": "b26268d6-8bb5-11e6-baff-00224d8226cd",
"type": "instance_disk",
"instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1"
}],
[{
"id": "b1c7544a-8bb5-11e6-850e-00224d8226cd",
"type": "instance_network_interface",
"instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1"
}],
]
search_params = {
'=': {
'instance_id': '9f9d01b9-4a58-4271-9e27-398b21ab20d1'
}
}
now = timeutils.utcnow()
self.useFixture(utils_fixture.TimeFixture(now))
expected_calls = [
mock.call.capabilities.list(),
mock.call.resource.search('instance_disk', search_params),
mock.call.resource.search('instance_network_interface',
search_params),
mock.call.resource.update('instance',
'9f9d01b9-4a58-4271-9e27-398b21ab20d1',
{'ended_at': now.isoformat()}),
mock.call.resource.update('instance_disk',
'b26268d6-8bb5-11e6-baff-00224d8226cd',
{'ended_at': now.isoformat()}),
mock.call.resource.update('instance_network_interface',
'b1c7544a-8bb5-11e6-850e-00224d8226cd',
{'ended_at': now.isoformat()}),
mock.call.resource.update('image',
'dc337359-de70-4044-8e2c-80573ba6e577',
{'ended_at': now.isoformat()}),
mock.call.resource.update('volume',
'6cc6e7dd-d17d-460f-ae79-7e08a216ce96',
{'ended_at': now.isoformat()}),
mock.call.resource.update('network',
'705e2c08-08e8-45cb-8673-5c5be955569b',
{'ended_at': now.isoformat()})
]
self.dispatcher.record_events([
INSTANCE_DELETE_START, IMAGE_DELETE_START, VOLUME_DELETE_START,
FLOATINGIP_DELETE_END
])
self.assertEqual(9, len(fakeclient.mock_calls))
for call in expected_calls:
self.assertIn(call, fakeclient.mock_calls)
def _do_test_activity_filter(self, expected_samples, fake_process_samples):
d = gnocchi.GnocchiDispatcher(self.conf.conf)
d.record_metering_data(self.samples)
fake_process_samples.assert_called_with(
mock.ANY, self.resource_id, 'disk.root.size',
expected_samples, True,
)
def _do_test_activity_filter(self, expected_measures, fake_batch, __):
d = gnocchi.GnocchiDispatcher(self.conf.conf)
d.record_metering_data(self.samples)
fake_batch.assert_called_with(mock.ANY, mock.ANY, {
'metrics': 1,
'resources': 1,
'measures': expected_measures
})
def setUp(self):
super(DispatcherWorkflowTest, self).setUp()
self.conf = self.useFixture(config_fixture.Config())
ks_client = mock.Mock(auth_token='fake_token')
ks_client.tenants.find.return_value = mock.Mock(
name='gnocchi', id='a2d42c23-d518-46b6-96ab-3fba2e146859')
self.useFixture(mockpatch.Patch(
'ceilometer.keystone_client.get_client',
return_value=ks_client))
ceilometer_service.prepare_service([])
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
self.sample['resource_id'] = str(uuid.uuid4())
def test_archive_policy_map_config(self):
archive_policy_map = yaml.dump({'foo.*': 'low'})
archive_policy_cfg_file = tempfile.NamedTemporaryFile(mode='w+b',
prefix="foo",
suffix=".yaml")
archive_policy_cfg_file.write(archive_policy_map.encode())
archive_policy_cfg_file.seek(0)
self.conf.conf.dispatcher_gnocchi.archive_policy_file = (
archive_policy_cfg_file.name)
d = gnocchi.GnocchiDispatcher(self.conf.conf)
legacy = d._load_archive_policy(self.conf.conf)
self.assertEqual(legacy.get('foo.disk.rate'), "low")
archive_policy_cfg_file.close()
def _do_test_activity_filter(self, expected_samples,
fake_process_resource):
def assert_samples(resource_id, metric_grouped_samples):
samples = []
for metric_name, s in metric_grouped_samples:
samples.extend(list(s))
self.assertEqual(expected_samples, samples)
fake_process_resource.side_effect = assert_samples
d = gnocchi.GnocchiDispatcher(self.conf.conf)
d.record_metering_data(self.samples)
fake_process_resource.assert_called_with(self.resource_id, mock.ANY)
def test_unhandled_meter(self, fake_batch):
samples = [{
'counter_name': 'unknown.meter',
'counter_unit': 'GB',
'counter_type': 'gauge',
'counter_volume': '2',
'user_id': 'test_user',
'project_id': 'test_project',
'source': 'openstack',
'timestamp': '2014-05-08 20:23:48.028195',
'resource_id': 'randomid',
'resource_metadata': {}
}]
d = gnocchi.GnocchiDispatcher(self.conf.conf)
d.record_metering_data(samples)
self.assertEqual(0, len(fake_batch.call_args[0][1]))
def test_event_workflow(self, fakeclient_cls):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
fakeclient.resource.search.side_effect = [
[{
"id": "b26268d6-8bb5-11e6-baff-00224d8226cd",
"type": "instance_disk",
"instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1"
}],
[{
"id": "b1c7544a-8bb5-11e6-850e-00224d8226cd",
"type": "instance_network_interface",
"instance_id": "9f9d01b9-4a58-4271-9e27-398b21ab20d1"
}],
]
search_params = {
'=': {
'instance_id': '9f9d01b9-4a58-4271-9e27-398b21ab20d1'
}
}
now = timeutils.utcnow()
self.useFixture(utils_fixture.TimeFixture(now))
expected_calls = [
mock.call.capabilities.list(),
mock.call.resource.search('instance_disk', search_params),
mock.call.resource.search('instance_network_interface',
search_params),
mock.call.resource.update('instance',
'9f9d01b9-4a58-4271-9e27-398b21ab20d1',
{'ended_at': now.isoformat()}),
mock.call.resource.update('instance_disk',
'b26268d6-8bb5-11e6-baff-00224d8226cd',
{'ended_at': now.isoformat()}),
mock.call.resource.update('instance_network_interface',
'b1c7544a-8bb5-11e6-850e-00224d8226cd',
{'ended_at': now.isoformat()})
]
self.dispatcher.record_events([INSTANCE_DELETE_START])
self.assertEqual(6, len(fakeclient.mock_calls))
for call in expected_calls:
self.assertIn(call, fakeclient.mock_calls)
def setUp(self):
super(DispatcherWorkflowTest, self).setUp()
self.conf = self.useFixture(config_fixture.Config())
# Set this explicitly to avoid conflicts with any existing
# configuration.
self.conf.config(url='http://localhost:8041',
group='dispatcher_gnocchi')
ks_client = mock.Mock(auth_token='fake_token')
ks_client.tenants.find.return_value = mock.Mock(
name='gnocchi', id='a2d42c23-d518-46b6-96ab-3fba2e146859')
self.useFixture(
mockpatch.Patch('ceilometer.keystone_client.get_client',
return_value=ks_client))
ceilometer_service.prepare_service([])
self.conf.config(resources_definition_file=self.path_get(
'etc/ceilometer/gnocchi_resources.yaml'),
group="dispatcher_gnocchi")
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
self.sample['resource_id'] = str(uuid.uuid4())
def test_workflow(self, fakeclient_cls, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
resource_id = self.sample['resource_id'].replace("/", "_")
metric_name = self.sample['counter_name']
gnocchi_id = uuid.uuid4()
expected_calls = [
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {
metric_name: self.measures_attributes
}},
create_metrics=True)
]
expected_debug = [
mock.call('gnocchi project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
]
measures_posted = False
batch_side_effect = []
if self.post_measure_fail:
batch_side_effect += [Exception('boom!')]
elif not self.resource_exists:
batch_side_effect += [
gnocchi_exc.BadRequest(
400, {
"cause":
"Unknown resources",
'detail': [{
'resource_id': gnocchi_id,
'original_resource_id': resource_id
}]
})
]
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
for k, v in six.iteritems(attributes['metrics']):
if k == 'disk.root.size':
v['unit'] = 'GB'
continue
if k == 'hardware.ipmi.node.power':
v['unit'] = 'W'
continue
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.create_resource_race:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
else: # not resource_exists
expected_debug.append(
mock.call('Resource %s created',
self.sample['resource_id']))
if not self.create_resource_fail:
expected_calls.append(
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {
metric_name: self.measures_attributes
}},
create_metrics=True))
if self.retry_post_measures_fail:
batch_side_effect += [Exception('boom!')]
else:
measures_posted = True
else:
measures_posted = True
if measures_posted:
batch_side_effect += [None]
expected_debug.append(
mock.call(
"%(measures)d measures posted against %(metrics)d "
"metrics through %(resources)d resources",
dict(measures=len(self.measures_attributes),
metrics=1,
resources=1)))
if self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.update_resource_fail:
fakeclient.resource.update.side_effect = [Exception('boom!')]
else:
expected_debug.append(
mock.call('Resource %s updated',
self.sample['resource_id']))
batch = fakeclient.metric.batch_resources_metrics_measures
batch.side_effect = batch_side_effect
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if (self.post_measure_fail or self.create_resource_fail
or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
else:
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fake_requests, client_logger, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
base_url = self.dispatcher.conf.dispatcher_gnocchi.url
url_params = {
'url': urlparse.urljoin(base_url, '/v1/resource'),
'resource_id': self.sample['resource_id'],
'resource_type': self.resource_type,
'metric_name': self.sample['counter_name']
}
headers = {
'Content-Type': 'application/json',
'X-Auth-Token': 'fake_token'
}
patch_responses = []
post_responses = []
# This is needed to mock Exception in py3
fake_requests.ConnectionError = requests.ConnectionError
expected_calls = [
mock.call.session(),
mock.call.adapters.HTTPAdapter(pool_block=True),
mock.call.session().mount('http://', mock.ANY),
mock.call.session().mount('https://', mock.ANY),
mock.call.session().post(
"%(url)s/%(resource_type)s/%(resource_id)s/"
"metric/%(metric_name)s/measures" % url_params,
headers=headers,
data=json_matcher(self.measures_attributes))
]
post_responses.append(MockResponse(self.measure))
if self.measure == 401:
type(self.ks_client).auth_token = mock.PropertyMock(side_effect=[
'fake_token', 'new_token', 'new_token', 'new_token',
'new_token'
])
headers = {
'Content-Type': 'application/json',
'X-Auth-Token': 'new_token'
}
expected_calls.append(mock.call.session().post(
"%(url)s/%(resource_type)s/%(resource_id)s/"
"metric/%(metric_name)s/measures" % url_params,
headers=headers,
data=json_matcher(self.measures_attributes)))
post_responses.append(MockResponse(200))
if self.post_resource:
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict((metric_name, {
'archive_policy_name': 'low'
}) for metric_name in self.metric_names)
expected_calls.append(
mock.call.session().post("%(url)s/%(resource_type)s" %
url_params,
headers=headers,
data=json_matcher(attributes)), )
post_responses.append(MockResponse(self.post_resource))
if self.metric:
expected_calls.append(mock.call.session().post(
"%(url)s/%(resource_type)s/%(resource_id)s/metric" %
url_params,
headers=headers,
data=json_matcher({
self.sample['counter_name']: {
'archive_policy_name': 'low'
}
})))
post_responses.append(MockResponse(self.metric))
if self.measure_retry:
expected_calls.append(mock.call.session().post(
"%(url)s/%(resource_type)s/%(resource_id)s/"
"metric/%(metric_name)s/measures" % url_params,
headers=headers,
data=json_matcher(self.measures_attributes)))
post_responses.append(MockResponse(self.measure_retry))
if self.patch_resource and self.patchable_attributes:
expected_calls.append(
mock.call.session().patch(
"%(url)s/%(resource_type)s/%(resource_id)s" % url_params,
headers=headers,
data=json_matcher(self.patchable_attributes)), )
patch_responses.append(MockResponse(self.patch_resource))
s = fake_requests.session.return_value
s.patch.side_effect = patch_responses
s.post.side_effect = post_responses
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if self.measure == 500 or self.measure_retry == 500:
logger.error.assert_called_with(
"Fail to post measure on metric %s of resource %s "
"with status: %d: Internal Server Error" %
(self.sample['counter_name'], self.sample['resource_id'], 500))
elif self.post_resource == 500 or (self.patch_resource == 500
and self.patchable_attributes):
logger.error.assert_called_with(
"Resource %s %s failed with status: "
"%d: Internal Server Error" %
(self.sample['resource_id'],
'update' if self.patch_resource else 'creation', 500))
elif self.metric == 500:
logger.error.assert_called_with(
"Fail to create metric %s of resource %s "
"with status: %d: Internal Server Error" %
(self.sample['counter_name'], self.sample['resource_id'], 500))
elif self.patch_resource == 204 and self.patchable_attributes:
client_logger.debug.assert_called_with('Resource %s updated',
self.sample['resource_id'])
elif self.measure == 200:
client_logger.debug.assert_called_with(
"Measure posted on metric %s of resource %s",
self.sample['counter_name'], self.sample['resource_id'])
self.assertEqual(expected_calls, fake_requests.mock_calls)
def test_workflow(self, fakeclient_cls, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
# FIXME(sileht): we don't use urlparse.quote here
# to ensure / is converted in %2F
# temporary disabled until we find a solution
# on gnocchi side. Current gnocchiclient doesn't
# encode the resource_id
resource_id = self.sample['resource_id'] # .replace("/", "%2F"),
metric_name = self.sample['counter_name']
gnocchi_id = gnocchi_utils.encode_resource_id(resource_id)
expected_calls = [
mock.call.capabilities.list(),
mock.call.metric.batch_resources_metrics_measures(
{gnocchi_id: {
metric_name: self.measures_attributes
}})
]
expected_debug = [
mock.call('gnocchi project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
]
measures_posted = False
batch_side_effect = []
if self.post_measure_fail:
batch_side_effect += [Exception('boom!')]
elif not self.resource_exists or not self.metric_exists:
batch_side_effect += [
gnocchi_exc.BadRequest(
400, "Unknown metrics: %s/%s" % (gnocchi_id, metric_name))
]
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.resource_exists:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
expected_calls.append(
mock.call.metric.create({
'name': self.sample['counter_name'],
'resource_id': resource_id
}))
if self.create_metric_fail:
fakeclient.metric.create.side_effect = [Exception('boom!')]
elif self.metric_exists:
fakeclient.metric.create.side_effect = [
gnocchi_exc.NamedMetricAreadyExists(409)
]
else:
fakeclient.metric.create.side_effect = [None]
else: # not resource_exists
expected_debug.append(
mock.call('Resource %s created',
self.sample['resource_id']))
if not self.create_resource_fail and not self.create_metric_fail:
expected_calls.append(
mock.call.metric.batch_resources_metrics_measures(
{gnocchi_id: {
metric_name: self.measures_attributes
}}))
if self.retry_post_measures_fail:
batch_side_effect += [Exception('boom!')]
else:
measures_posted = True
else:
measures_posted = True
if measures_posted:
batch_side_effect += [None]
expected_debug.append(
mock.call(
"%(measures)d measures posted against %(metrics)d "
"metrics through %(resources)d resources",
dict(measures=len(self.measures_attributes),
metrics=1,
resources=1)))
if self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.update_resource_fail:
fakeclient.resource.update.side_effect = [Exception('boom!')]
else:
expected_debug.append(
mock.call('Resource %s updated',
self.sample['resource_id']))
batch = fakeclient.metric.batch_resources_metrics_measures
batch.side_effect = batch_side_effect
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if (self.post_measure_fail or self.create_metric_fail
or self.create_resource_fail or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
else:
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_archive_policy_default(self):
d = gnocchi.GnocchiDispatcher(self.conf.conf)
self.assertEqual(d.gnocchi_archive_policy_default, "low")
def test_workflow(self, fakeclient_cls, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
# FIXME(sileht): we don't use urlparse.quote here
# to ensure / is converted in %2F
# temporary disabled until we find a solution
# on gnocchi side. Current gnocchiclient doesn't
# encode the resource_id
resource_id = self.sample['resource_id'] # .replace("/", "%2F"),
metric_name = self.sample['counter_name']
expected_calls = [
mock.call.capabilities.list(),
mock.call.metric.add_measures(metric_name,
self.measures_attributes,
resource_id)
]
add_measures_side_effect = []
if self.measure == 404 and self.post_resource:
add_measures_side_effect += [gnocchi_exc.ResourceNotFound(404)]
elif self.measure == 404 and self.metric:
add_measures_side_effect += [gnocchi_exc.MetricNotFound(404)]
elif self.measure == 500:
add_measures_side_effect += [Exception('boom!')]
if self.post_resource:
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.post_resource == 409:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
elif self.post_resource == 500:
fakeclient.resource.create.side_effect = [Exception('boom!')]
if self.metric:
expected_calls.append(
mock.call.metric.create({
'name': self.sample['counter_name'],
'resource_id': resource_id
}))
if self.metric == 409:
fakeclient.metric.create.side_effect = [
gnocchi_exc.NamedMetricAreadyExists(409)
]
elif self.metric == 500:
fakeclient.metric.create.side_effect = [Exception('boom!')]
if self.measure_retry:
expected_calls.append(
mock.call.metric.add_measures(metric_name,
self.measures_attributes,
resource_id))
if self.measure_retry == 204:
add_measures_side_effect += [None]
elif self.measure_retry == 500:
add_measures_side_effect += [Exception('boom!')]
else:
add_measures_side_effect += [None]
if self.patch_resource and self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.patch_resource == 500:
fakeclient.resource.update.side_effect = [Exception('boom!')]
fakeclient.metric.add_measures.side_effect = add_measures_side_effect
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if (self.measure == 500 or self.measure_retry == 500
or self.metric == 500 or self.post_resource == 500
or (self.patch_resource == 500 and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
elif self.patch_resource == 204 and self.patchable_attributes:
logger.debug.assert_called_with('Resource %s updated',
self.sample['resource_id'])
self.assertEqual(0, logger.error.call_count)
elif self.measure == 200:
logger.debug.assert_called_with(
"Measure posted on metric %s of resource %s",
self.sample['counter_name'], self.sample['resource_id'])
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
def test_extensions_load(self):
self.conf.config(filter_service_activity=False,
group='dispatcher_gnocchi')
d = gnocchi.GnocchiDispatcher(self.conf.conf)
self.assertIn('instance', d.mgmr.names())
self.assertIn('volume', d.mgmr.names())