def _if_not_cached(self, operation, resource_type, resource, method, *args,
**kwargs):
if self.cache:
cache_key = resource['id']
attribute_hash = self._check_resource_cache(cache_key, resource)
hit = False
if attribute_hash:
with self._gnocchi_resource_lock[cache_key]:
# NOTE(luogangyi): there is a possibility that the
# resource was already built in cache by another
# ceilometer-collector when we get the lock here.
attribute_hash = self._check_resource_cache(
cache_key, resource)
if attribute_hash:
method(resource_type, resource, *args, **kwargs)
self.cache.set(cache_key, attribute_hash)
else:
hit = True
LOG.debug('resource cache recheck hit for '
'%s %s', operation, cache_key)
self._gnocchi_resource_lock.pop(cache_key, None)
else:
hit = True
LOG.debug('Resource cache hit for %s %s', operation, cache_key)
if hit and operation == "create":
raise gnocchi_exc.ResourceAlreadyExists()
else:
method(resource_type, resource, *args, **kwargs)
def test_workflow(self, fakeclient_cls, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
resource_id = self.sample['resource_id'].replace("/", "_")
metric_name = self.sample['counter_name']
gnocchi_id = uuid.uuid4()
expected_calls = [
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {
metric_name: self.measures_attributes
}},
create_metrics=True)
]
expected_debug = [
mock.call('gnocchi project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
]
measures_posted = False
batch_side_effect = []
if self.post_measure_fail:
batch_side_effect += [Exception('boom!')]
elif not self.resource_exists:
batch_side_effect += [
gnocchi_exc.BadRequest(
400, {
"cause":
"Unknown resources",
'detail': [{
'resource_id': gnocchi_id,
'original_resource_id': resource_id
}]
})
]
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
for k, v in six.iteritems(attributes['metrics']):
if k == 'disk.root.size':
v['unit'] = 'GB'
continue
if k == 'hardware.ipmi.node.power':
v['unit'] = 'W'
continue
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.create_resource_race:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
else: # not resource_exists
expected_debug.append(
mock.call('Resource %s created',
self.sample['resource_id']))
if not self.create_resource_fail:
expected_calls.append(
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {
metric_name: self.measures_attributes
}},
create_metrics=True))
if self.retry_post_measures_fail:
batch_side_effect += [Exception('boom!')]
else:
measures_posted = True
else:
measures_posted = True
if measures_posted:
batch_side_effect += [None]
expected_debug.append(
mock.call(
"%(measures)d measures posted against %(metrics)d "
"metrics through %(resources)d resources",
dict(measures=len(self.measures_attributes),
metrics=1,
resources=1)))
if self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.update_resource_fail:
fakeclient.resource.update.side_effect = [Exception('boom!')]
else:
expected_debug.append(
mock.call('Resource %s updated',
self.sample['resource_id']))
batch = fakeclient.metric.batch_resources_metrics_measures
batch.side_effect = batch_side_effect
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if (self.post_measure_fail or self.create_resource_fail
or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
else:
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
# FIXME(sileht): we don't use urlparse.quote here
# to ensure / is converted in %2F
# temporary disabled until we find a solution
# on gnocchi side. Current gnocchiclient doesn't
# encode the resource_id
resource_id = self.sample['resource_id'] # .replace("/", "%2F"),
metric_name = self.sample['counter_name']
gnocchi_id = gnocchi_utils.encode_resource_id(resource_id)
expected_calls = [
mock.call.capabilities.list(),
mock.call.metric.batch_resources_metrics_measures(
{gnocchi_id: {
metric_name: self.measures_attributes
}})
]
expected_debug = [
mock.call('gnocchi project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
]
measures_posted = False
batch_side_effect = []
if self.post_measure_fail:
batch_side_effect += [Exception('boom!')]
elif not self.resource_exists or not self.metric_exists:
batch_side_effect += [
gnocchi_exc.BadRequest(
400, "Unknown metrics: %s/%s" % (gnocchi_id, metric_name))
]
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.resource_exists:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
expected_calls.append(
mock.call.metric.create({
'name': self.sample['counter_name'],
'resource_id': resource_id
}))
if self.create_metric_fail:
fakeclient.metric.create.side_effect = [Exception('boom!')]
elif self.metric_exists:
fakeclient.metric.create.side_effect = [
gnocchi_exc.NamedMetricAreadyExists(409)
]
else:
fakeclient.metric.create.side_effect = [None]
else: # not resource_exists
expected_debug.append(
mock.call('Resource %s created',
self.sample['resource_id']))
if not self.create_resource_fail and not self.create_metric_fail:
expected_calls.append(
mock.call.metric.batch_resources_metrics_measures(
{gnocchi_id: {
metric_name: self.measures_attributes
}}))
if self.retry_post_measures_fail:
batch_side_effect += [Exception('boom!')]
else:
measures_posted = True
else:
measures_posted = True
if measures_posted:
batch_side_effect += [None]
expected_debug.append(
mock.call(
"%(measures)d measures posted against %(metrics)d "
"metrics through %(resources)d resources",
dict(measures=len(self.measures_attributes),
metrics=1,
resources=1)))
if self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.update_resource_fail:
fakeclient.resource.update.side_effect = [Exception('boom!')]
else:
expected_debug.append(
mock.call('Resource %s updated',
self.sample['resource_id']))
batch = fakeclient.metric.batch_resources_metrics_measures
batch.side_effect = batch_side_effect
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if (self.post_measure_fail or self.create_metric_fail
or self.create_resource_fail or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
else:
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger):
fakeclient = fakeclient_cls.return_value
resource_id = self.sample.resource_id.replace("/", "_")
metric_name = self.sample.name
gnocchi_id = uuid.uuid4()
expected_calls = [
mock.call.archive_policy.create({"name": "ceilometer-low",
"back_window": 0,
"aggregation_methods": ["mean"],
"definition": mock.ANY}),
mock.call.archive_policy.create({"name": "ceilometer-low-rate",
"back_window": 0,
"aggregation_methods": [
"mean", "rate:mean"],
"definition": mock.ANY}),
mock.call.archive_policy.create({"name": "ceilometer-high",
"back_window": 0,
"aggregation_methods": ["mean"],
"definition": mock.ANY}),
mock.call.archive_policy.create({"name": "ceilometer-high-rate",
"back_window": 0,
"aggregation_methods": [
"mean", "rate:mean"],
"definition": mock.ANY}),
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {metric_name: self.metric_attributes}},
create_metrics=True)
]
expected_debug = [
mock.call('filtered project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
mock.call('Processing sample [%s] for resource ID [%s].',
self.sample, resource_id),
]
measures_posted = False
batch_side_effect = []
if self.post_measure_fail:
batch_side_effect += [Exception('boom!')]
elif not self.resource_exists:
batch_side_effect += [
gnocchi_exc.BadRequest(
400, {"cause": "Unknown resources",
'detail': [{
'resource_id': gnocchi_id,
'original_resource_id': resource_id}]})]
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample.resource_id
expected_calls.append(mock.call.resource.create(
self.resource_type, attributes))
if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.create_resource_race:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)]
else: # not resource_exists
expected_debug.append(mock.call(
'Resource %s created', self.sample.resource_id))
if not self.create_resource_fail:
expected_calls.append(
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {metric_name: self.metric_attributes}},
create_metrics=True)
)
if self.retry_post_measures_fail:
batch_side_effect += [Exception('boom!')]
else:
measures_posted = True
else:
measures_posted = True
if measures_posted:
batch_side_effect += [None]
expected_debug.append(
mock.call("%d measures posted against %d metrics through %d "
"resources", len(self.metric_attributes["measures"]),
1, 1)
)
if self.patchable_attributes:
expected_calls.append(mock.call.resource.update(
self.resource_type, resource_id,
self.patchable_attributes))
if self.update_resource_fail:
fakeclient.resource.update.side_effect = [Exception('boom!')]
else:
expected_debug.append(mock.call(
'Resource %s updated', self.sample.resource_id))
batch = fakeclient.metric.batch_resources_metrics_measures
batch.side_effect = batch_side_effect
url = netutils.urlsplit("gnocchi://")
publisher = gnocchi.GnocchiPublisher(self.conf.conf, url)
publisher.publish_samples([self.sample])
# Check that the last log message is the expected one
if (self.post_measure_fail
or self.create_resource_fail
or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
else:
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger):
fakeclient = fakeclient_cls.return_value
resource_id = self.sample.resource_id.replace("/", "_")
metric_name = self.sample.name
gnocchi_id = uuid.uuid4()
expected_calls = [
mock.call.archive_policy.get("ceilometer-low"),
mock.call.archive_policy.get("ceilometer-low-rate"),
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {
metric_name: self.metric_attributes
}},
create_metrics=True)
]
expected_debug = [
mock.call('filtered project found: %s',
'a2d42c23-d518-46b6-96ab-3fba2e146859'),
]
measures_posted = False
batch_side_effect = []
if self.post_measure_fail:
batch_side_effect += [Exception('boom!')]
elif not self.resource_exists:
batch_side_effect += [
gnocchi_exc.BadRequest(
400, {
"cause":
"Unknown resources",
'detail': [{
'resource_id': gnocchi_id,
'original_resource_id': resource_id
}]
})
]
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample.resource_id
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
for k, v in six.iteritems(attributes['metrics']):
if k in [
"cpu", "disk.read.requests", "disk.write.requests",
"disk.read.bytes", "disk.write.bytes"
]:
v["archive_policy_name"] = "ceilometer-low-rate"
else:
v["archive_policy_name"] = "ceilometer-low"
if k == 'disk.root.size':
v['unit'] = 'GB'
elif k == 'hardware.ipmi.node.power':
v['unit'] = 'W'
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.create_resource_fail:
fakeclient.resource.create.side_effect = [Exception('boom!')]
elif self.create_resource_race:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
else: # not resource_exists
expected_debug.append(
mock.call('Resource %s created', self.sample.resource_id))
if not self.create_resource_fail:
expected_calls.append(
mock.call.metric.batch_resources_metrics_measures(
{resource_id: {
metric_name: self.metric_attributes
}},
create_metrics=True))
if self.retry_post_measures_fail:
batch_side_effect += [Exception('boom!')]
else:
measures_posted = True
else:
measures_posted = True
if measures_posted:
batch_side_effect += [None]
expected_debug.append(
mock.call(
"%d measures posted against %d metrics through %d "
"resources", len(self.metric_attributes["measures"]), 1,
1))
if self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.update_resource_fail:
fakeclient.resource.update.side_effect = [Exception('boom!')]
else:
expected_debug.append(
mock.call('Resource %s updated', self.sample.resource_id))
batch = fakeclient.metric.batch_resources_metrics_measures
batch.side_effect = batch_side_effect
url = netutils.urlsplit("gnocchi://")
publisher = gnocchi.GnocchiPublisher(self.conf.conf, url)
publisher.publish_samples([self.sample])
# Check that the last log message is the expected one
if (self.post_measure_fail or self.create_resource_fail
or self.retry_post_measures_fail
or (self.update_resource_fail and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
else:
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger):
self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf)
fakeclient = fakeclient_cls.return_value
# FIXME(sileht): we don't use urlparse.quote here
# to ensure / is converted in %2F
# temporary disabled until we find a solution
# on gnocchi side. Current gnocchiclient doesn't
# encode the resource_id
resource_id = self.sample['resource_id'] # .replace("/", "%2F"),
metric_name = self.sample['counter_name']
expected_calls = [
mock.call.capabilities.list(),
mock.call.metric.add_measures(metric_name,
self.measures_attributes,
resource_id)
]
add_measures_side_effect = []
if self.measure == 404 and self.post_resource:
add_measures_side_effect += [gnocchi_exc.ResourceNotFound(404)]
elif self.measure == 404 and self.metric:
add_measures_side_effect += [gnocchi_exc.MetricNotFound(404)]
elif self.measure == 500:
add_measures_side_effect += [Exception('boom!')]
if self.post_resource:
attributes = self.postable_attributes.copy()
attributes.update(self.patchable_attributes)
attributes['id'] = self.sample['resource_id']
attributes['metrics'] = dict(
(metric_name, {}) for metric_name in self.metric_names)
expected_calls.append(
mock.call.resource.create(self.resource_type, attributes))
if self.post_resource == 409:
fakeclient.resource.create.side_effect = [
gnocchi_exc.ResourceAlreadyExists(409)
]
elif self.post_resource == 500:
fakeclient.resource.create.side_effect = [Exception('boom!')]
if self.metric:
expected_calls.append(
mock.call.metric.create({
'name': self.sample['counter_name'],
'resource_id': resource_id
}))
if self.metric == 409:
fakeclient.metric.create.side_effect = [
gnocchi_exc.NamedMetricAreadyExists(409)
]
elif self.metric == 500:
fakeclient.metric.create.side_effect = [Exception('boom!')]
if self.measure_retry:
expected_calls.append(
mock.call.metric.add_measures(metric_name,
self.measures_attributes,
resource_id))
if self.measure_retry == 204:
add_measures_side_effect += [None]
elif self.measure_retry == 500:
add_measures_side_effect += [Exception('boom!')]
else:
add_measures_side_effect += [None]
if self.patch_resource and self.patchable_attributes:
expected_calls.append(
mock.call.resource.update(self.resource_type, resource_id,
self.patchable_attributes))
if self.patch_resource == 500:
fakeclient.resource.update.side_effect = [Exception('boom!')]
fakeclient.metric.add_measures.side_effect = add_measures_side_effect
self.dispatcher.record_metering_data([self.sample])
# Check that the last log message is the expected one
if (self.measure == 500 or self.measure_retry == 500
or self.metric == 500 or self.post_resource == 500
or (self.patch_resource == 500 and self.patchable_attributes)):
logger.error.assert_called_with('boom!', exc_info=True)
elif self.patch_resource == 204 and self.patchable_attributes:
logger.debug.assert_called_with('Resource %s updated',
self.sample['resource_id'])
self.assertEqual(0, logger.error.call_count)
elif self.measure == 200:
logger.debug.assert_called_with(
"Measure posted on metric %s of resource %s",
self.sample['counter_name'], self.sample['resource_id'])
self.assertEqual(0, logger.error.call_count)
self.assertEqual(expected_calls, fakeclient.mock_calls)
