def _if_not_cached(self, operation, resource_type, resource, method, *args, **kwargs): if self.cache: cache_key = resource['id'] attribute_hash = self._check_resource_cache(cache_key, resource) hit = False if attribute_hash: with self._gnocchi_resource_lock[cache_key]: # NOTE(luogangyi): there is a possibility that the # resource was already built in cache by another # ceilometer-collector when we get the lock here. attribute_hash = self._check_resource_cache( cache_key, resource) if attribute_hash: method(resource_type, resource, *args, **kwargs) self.cache.set(cache_key, attribute_hash) else: hit = True LOG.debug('resource cache recheck hit for ' '%s %s', operation, cache_key) self._gnocchi_resource_lock.pop(cache_key, None) else: hit = True LOG.debug('Resource cache hit for %s %s', operation, cache_key) if hit and operation == "create": raise gnocchi_exc.ResourceAlreadyExists() else: method(resource_type, resource, *args, **kwargs)
def test_workflow(self, fakeclient_cls, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value resource_id = self.sample['resource_id'].replace("/", "_") metric_name = self.sample['counter_name'] gnocchi_id = uuid.uuid4() expected_calls = [ mock.call.metric.batch_resources_metrics_measures( {resource_id: { metric_name: self.measures_attributes }}, create_metrics=True) ] expected_debug = [ mock.call('gnocchi project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), ] measures_posted = False batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] elif not self.resource_exists: batch_side_effect += [ gnocchi_exc.BadRequest( 400, { "cause": "Unknown resources", 'detail': [{ 'resource_id': gnocchi_id, 'original_resource_id': resource_id }] }) ] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) for k, v in six.iteritems(attributes['metrics']): if k == 'disk.root.size': v['unit'] = 'GB' continue if k == 'hardware.ipmi.node.power': v['unit'] = 'W' continue expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] elif self.create_resource_race: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] else: # not resource_exists expected_debug.append( mock.call('Resource %s created', self.sample['resource_id'])) if not self.create_resource_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( {resource_id: { metric_name: self.measures_attributes }}, create_metrics=True)) if self.retry_post_measures_fail: batch_side_effect += [Exception('boom!')] else: measures_posted = True else: measures_posted = True if measures_posted: batch_side_effect += [None] expected_debug.append( mock.call( "%(measures)d measures posted against %(metrics)d " "metrics through %(resources)d resources", dict(measures=len(self.measures_attributes), metrics=1, resources=1))) if self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.update_resource_fail: fakeclient.resource.update.side_effect = [Exception('boom!')] else: expected_debug.append( mock.call('Resource %s updated', self.sample['resource_id'])) batch = fakeclient.metric.batch_resources_metrics_measures batch.side_effect = batch_side_effect self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if (self.post_measure_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) else: self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls) self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value # FIXME(sileht): we don't use urlparse.quote here # to ensure / is converted in %2F # temporary disabled until we find a solution # on gnocchi side. Current gnocchiclient doesn't # encode the resource_id resource_id = self.sample['resource_id'] # .replace("/", "%2F"), metric_name = self.sample['counter_name'] gnocchi_id = gnocchi_utils.encode_resource_id(resource_id) expected_calls = [ mock.call.capabilities.list(), mock.call.metric.batch_resources_metrics_measures( {gnocchi_id: { metric_name: self.measures_attributes }}) ] expected_debug = [ mock.call('gnocchi project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), ] measures_posted = False batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] elif not self.resource_exists or not self.metric_exists: batch_side_effect += [ gnocchi_exc.BadRequest( 400, "Unknown metrics: %s/%s" % (gnocchi_id, metric_name)) ] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] elif self.resource_exists: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] expected_calls.append( mock.call.metric.create({ 'name': self.sample['counter_name'], 'resource_id': resource_id })) if self.create_metric_fail: fakeclient.metric.create.side_effect = [Exception('boom!')] elif self.metric_exists: fakeclient.metric.create.side_effect = [ gnocchi_exc.NamedMetricAreadyExists(409) ] else: fakeclient.metric.create.side_effect = [None] else: # not resource_exists expected_debug.append( mock.call('Resource %s created', self.sample['resource_id'])) if not self.create_resource_fail and not self.create_metric_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( {gnocchi_id: { metric_name: self.measures_attributes }})) if self.retry_post_measures_fail: batch_side_effect += [Exception('boom!')] else: measures_posted = True else: measures_posted = True if measures_posted: batch_side_effect += [None] expected_debug.append( mock.call( "%(measures)d measures posted against %(metrics)d " "metrics through %(resources)d resources", dict(measures=len(self.measures_attributes), metrics=1, resources=1))) if self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.update_resource_fail: fakeclient.resource.update.side_effect = [Exception('boom!')] else: expected_debug.append( mock.call('Resource %s updated', self.sample['resource_id'])) batch = fakeclient.metric.batch_resources_metrics_measures batch.side_effect = batch_side_effect self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if (self.post_measure_fail or self.create_metric_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) else: self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls) self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger): fakeclient = fakeclient_cls.return_value resource_id = self.sample.resource_id.replace("/", "_") metric_name = self.sample.name gnocchi_id = uuid.uuid4() expected_calls = [ mock.call.archive_policy.create({"name": "ceilometer-low", "back_window": 0, "aggregation_methods": ["mean"], "definition": mock.ANY}), mock.call.archive_policy.create({"name": "ceilometer-low-rate", "back_window": 0, "aggregation_methods": [ "mean", "rate:mean"], "definition": mock.ANY}), mock.call.archive_policy.create({"name": "ceilometer-high", "back_window": 0, "aggregation_methods": ["mean"], "definition": mock.ANY}), mock.call.archive_policy.create({"name": "ceilometer-high-rate", "back_window": 0, "aggregation_methods": [ "mean", "rate:mean"], "definition": mock.ANY}), mock.call.metric.batch_resources_metrics_measures( {resource_id: {metric_name: self.metric_attributes}}, create_metrics=True) ] expected_debug = [ mock.call('filtered project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), mock.call('Processing sample [%s] for resource ID [%s].', self.sample, resource_id), ] measures_posted = False batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] elif not self.resource_exists: batch_side_effect += [ gnocchi_exc.BadRequest( 400, {"cause": "Unknown resources", 'detail': [{ 'resource_id': gnocchi_id, 'original_resource_id': resource_id}]})] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample.resource_id expected_calls.append(mock.call.resource.create( self.resource_type, attributes)) if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] elif self.create_resource_race: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409)] else: # not resource_exists expected_debug.append(mock.call( 'Resource %s created', self.sample.resource_id)) if not self.create_resource_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( {resource_id: {metric_name: self.metric_attributes}}, create_metrics=True) ) if self.retry_post_measures_fail: batch_side_effect += [Exception('boom!')] else: measures_posted = True else: measures_posted = True if measures_posted: batch_side_effect += [None] expected_debug.append( mock.call("%d measures posted against %d metrics through %d " "resources", len(self.metric_attributes["measures"]), 1, 1) ) if self.patchable_attributes: expected_calls.append(mock.call.resource.update( self.resource_type, resource_id, self.patchable_attributes)) if self.update_resource_fail: fakeclient.resource.update.side_effect = [Exception('boom!')] else: expected_debug.append(mock.call( 'Resource %s updated', self.sample.resource_id)) batch = fakeclient.metric.batch_resources_metrics_measures batch.side_effect = batch_side_effect url = netutils.urlsplit("gnocchi://") publisher = gnocchi.GnocchiPublisher(self.conf.conf, url) publisher.publish_samples([self.sample]) # Check that the last log message is the expected one if (self.post_measure_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) else: self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls) self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger): fakeclient = fakeclient_cls.return_value resource_id = self.sample.resource_id.replace("/", "_") metric_name = self.sample.name gnocchi_id = uuid.uuid4() expected_calls = [ mock.call.archive_policy.get("ceilometer-low"), mock.call.archive_policy.get("ceilometer-low-rate"), mock.call.metric.batch_resources_metrics_measures( {resource_id: { metric_name: self.metric_attributes }}, create_metrics=True) ] expected_debug = [ mock.call('filtered project found: %s', 'a2d42c23-d518-46b6-96ab-3fba2e146859'), ] measures_posted = False batch_side_effect = [] if self.post_measure_fail: batch_side_effect += [Exception('boom!')] elif not self.resource_exists: batch_side_effect += [ gnocchi_exc.BadRequest( 400, { "cause": "Unknown resources", 'detail': [{ 'resource_id': gnocchi_id, 'original_resource_id': resource_id }] }) ] attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample.resource_id attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) for k, v in six.iteritems(attributes['metrics']): if k in [ "cpu", "disk.read.requests", "disk.write.requests", "disk.read.bytes", "disk.write.bytes" ]: v["archive_policy_name"] = "ceilometer-low-rate" else: v["archive_policy_name"] = "ceilometer-low" if k == 'disk.root.size': v['unit'] = 'GB' elif k == 'hardware.ipmi.node.power': v['unit'] = 'W' expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.create_resource_fail: fakeclient.resource.create.side_effect = [Exception('boom!')] elif self.create_resource_race: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] else: # not resource_exists expected_debug.append( mock.call('Resource %s created', self.sample.resource_id)) if not self.create_resource_fail: expected_calls.append( mock.call.metric.batch_resources_metrics_measures( {resource_id: { metric_name: self.metric_attributes }}, create_metrics=True)) if self.retry_post_measures_fail: batch_side_effect += [Exception('boom!')] else: measures_posted = True else: measures_posted = True if measures_posted: batch_side_effect += [None] expected_debug.append( mock.call( "%d measures posted against %d metrics through %d " "resources", len(self.metric_attributes["measures"]), 1, 1)) if self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.update_resource_fail: fakeclient.resource.update.side_effect = [Exception('boom!')] else: expected_debug.append( mock.call('Resource %s updated', self.sample.resource_id)) batch = fakeclient.metric.batch_resources_metrics_measures batch.side_effect = batch_side_effect url = netutils.urlsplit("gnocchi://") publisher = gnocchi.GnocchiPublisher(self.conf.conf, url) publisher.publish_samples([self.sample]) # Check that the last log message is the expected one if (self.post_measure_fail or self.create_resource_fail or self.retry_post_measures_fail or (self.update_resource_fail and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) else: self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls) self.assertEqual(expected_debug, logger.debug.mock_calls)
def test_workflow(self, fakeclient_cls, logger): self.dispatcher = gnocchi.GnocchiDispatcher(self.conf.conf) fakeclient = fakeclient_cls.return_value # FIXME(sileht): we don't use urlparse.quote here # to ensure / is converted in %2F # temporary disabled until we find a solution # on gnocchi side. Current gnocchiclient doesn't # encode the resource_id resource_id = self.sample['resource_id'] # .replace("/", "%2F"), metric_name = self.sample['counter_name'] expected_calls = [ mock.call.capabilities.list(), mock.call.metric.add_measures(metric_name, self.measures_attributes, resource_id) ] add_measures_side_effect = [] if self.measure == 404 and self.post_resource: add_measures_side_effect += [gnocchi_exc.ResourceNotFound(404)] elif self.measure == 404 and self.metric: add_measures_side_effect += [gnocchi_exc.MetricNotFound(404)] elif self.measure == 500: add_measures_side_effect += [Exception('boom!')] if self.post_resource: attributes = self.postable_attributes.copy() attributes.update(self.patchable_attributes) attributes['id'] = self.sample['resource_id'] attributes['metrics'] = dict( (metric_name, {}) for metric_name in self.metric_names) expected_calls.append( mock.call.resource.create(self.resource_type, attributes)) if self.post_resource == 409: fakeclient.resource.create.side_effect = [ gnocchi_exc.ResourceAlreadyExists(409) ] elif self.post_resource == 500: fakeclient.resource.create.side_effect = [Exception('boom!')] if self.metric: expected_calls.append( mock.call.metric.create({ 'name': self.sample['counter_name'], 'resource_id': resource_id })) if self.metric == 409: fakeclient.metric.create.side_effect = [ gnocchi_exc.NamedMetricAreadyExists(409) ] elif self.metric == 500: fakeclient.metric.create.side_effect = [Exception('boom!')] if self.measure_retry: expected_calls.append( mock.call.metric.add_measures(metric_name, self.measures_attributes, resource_id)) if self.measure_retry == 204: add_measures_side_effect += [None] elif self.measure_retry == 500: add_measures_side_effect += [Exception('boom!')] else: add_measures_side_effect += [None] if self.patch_resource and self.patchable_attributes: expected_calls.append( mock.call.resource.update(self.resource_type, resource_id, self.patchable_attributes)) if self.patch_resource == 500: fakeclient.resource.update.side_effect = [Exception('boom!')] fakeclient.metric.add_measures.side_effect = add_measures_side_effect self.dispatcher.record_metering_data([self.sample]) # Check that the last log message is the expected one if (self.measure == 500 or self.measure_retry == 500 or self.metric == 500 or self.post_resource == 500 or (self.patch_resource == 500 and self.patchable_attributes)): logger.error.assert_called_with('boom!', exc_info=True) elif self.patch_resource == 204 and self.patchable_attributes: logger.debug.assert_called_with('Resource %s updated', self.sample['resource_id']) self.assertEqual(0, logger.error.call_count) elif self.measure == 200: logger.debug.assert_called_with( "Measure posted on metric %s of resource %s", self.sample['counter_name'], self.sample['resource_id']) self.assertEqual(0, logger.error.call_count) self.assertEqual(expected_calls, fakeclient.mock_calls)