def _expand_metrics(self, resource, resource_id, mappings, start, end,
resource_type):
for mapping in mappings:
name, statistics = list(mapping.items())[0]
qty = self._get_resource_qty(
name,
start,
end,
resource_id,
statistics,
)
try:
conv_data = METRICS_CONF['metrics_units'][resource_type]
conv_data = conv_data.get(name)
if conv_data:
resource[name] = ck_utils.convert_unit(
qty,
conv_data.get('factor', 1),
conv_data.get('offset', 0),
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning(
'Error when trying to use yaml metrology conf.\n'
'Fallback on the deprecated hardcoded dict method.')
names = ['network.outgoing.bytes', 'network.incoming.bytes']
if name in names:
qty = qty / units.M
elif 'image.' in name:
qty = qty / units.Mi
resource[name] = qty
def _format_data(self, metric_name, scope_key, scope_id, start, end, data):
"""Formats Prometheus data format to Cloudkitty data format.
Returns metadata, groupby, qty
"""
metadata = {}
for meta in self.conf[metric_name]['metadata']:
metadata[meta] = data['metric'].get(meta, '')
groupby = {scope_key: scope_id}
for meta in self.conf[metric_name]['groupby']:
groupby[meta] = data['metric'].get(meta, '')
with localcontext() as ctx:
ctx.prec = 9
ctx.rounding = ROUND_HALF_UP
qty = ck_utils.convert_unit(
+Decimal(data['value'][1]),
self.conf[metric_name]['factor'],
self.conf[metric_name]['offset'],
)
qty = ck_utils.mutate(qty, self.conf[metric_name]['mutate'])
return metadata, groupby, qty
def _get_network_bw(self,
direction,
start,
end=None,
project_id=None,
q_filter=None):
if direction == 'in':
resource_type = 'network.incoming.bytes'
else:
direction = 'out'
resource_type = 'network.outgoing.bytes'
active_tap_stats = self.resources_stats(resource_type, start, end,
project_id, q_filter)
bw_data = []
for tap_stat in active_tap_stats:
tap_id = tap_stat.groupby['resource_id']
if not self._cacher.has_resource_detail('network.tap', tap_id):
raw_resource = self._conn.resources.get(tap_id)
tap = self.t_ceilometer.strip_resource_data(
'network.tap', raw_resource)
self._cacher.add_resource_detail('network.tap', tap_id, tap)
tap = self._cacher.get_resource_detail('network.tap', tap_id)
# Unit conversion
try:
conv = METRICS_CONF['metrics_units']['network.bw.' + direction]
tap_bw_mb = ck_utils.convert_unit(
decimal.Decimal(tap_stat.max),
conv[resource_type].get('factor', 1),
conv[resource_type].get('offset', 0),
)
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated hardcoded method.')
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
try:
met = METRICS_CONF['metrics_units']['network.bw.' + direction]
bw_data.append(
self.t_cloudkitty.format_item(
tap,
list(met.values())[0]['unit'],
tap_bw_mb,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
bw_data.append(
self.t_cloudkitty.format_item(
tap,
self.units_mappings['network.bw.' + direction],
tap_bw_mb,
))
ck_res_name = 'network.bw.{}'.format(direction)
if not bw_data:
raise collector.NoDataCollected(self.collector_name, ck_res_name)
return self.t_cloudkitty.format_service(ck_res_name, bw_data)
def resource_info(self, resource_name, start, end,
project_id, q_filter=None):
met = self.conf['metrics'][resource_name]
unit = met['unit']
qty = 1 if met.get('countable_unit') else met['resource']
resources = self.get_resources(
resource_name,
start,
end,
project_id=project_id,
q_filter=q_filter,
)
formated_resources = list()
for resource in resources:
resource_data = self.t_gnocchi.strip_resource_data(
resource_name, resource)
mapp = self.conf['metrics'][resource_name]['aggregation_method']
self._expand_metrics(
[resource_data],
mapp,
start,
end,
resource_name,
)
resource_data.pop('metrics', None)
# Unit conversion
if isinstance(qty, str):
resource_data[resource_name] = ck_utils.convert_unit(
resource_data[resource_name],
self.conf['metrics'][resource_name].get('factor', 1),
self.conf['metrics'][resource_name].get('offset', 0),
)
val = qty if isinstance(qty, int) else resource_data[resource_name]
data = self.t_cloudkitty.format_item(
resource_data,
unit,
decimal.Decimal(val)
)
# NOTE(sheeprine): Reference to gnocchi resource used by storage
data['resource_id'] = data['desc']['resource_id']
formated_resources.append(data)
return formated_resources
def get_image(self, start, end=None, project_id=None, q_filter=None):
active_image_stats = self.resources_stats('image.size', start, end,
project_id, q_filter)
image_data = []
for image_stats in active_image_stats:
image_id = image_stats.groupby['resource_id']
if not self._cacher.has_resource_detail('image', image_id):
raw_resource = self._conn.resources.get(image_id)
image = self.t_ceilometer.strip_resource_data(
'image', raw_resource)
self._cacher.add_resource_detail('image', image_id, image)
image = self._cacher.get_resource_detail('image', image_id)
# Unit conversion
try:
conv_data = METRICS_CONF['metrics_units']['image']
image_size_mb = ck_utils.convert_unit(
decimal.Decimal(image_stats.max),
conv_data['image.size'].get('factor', 1),
conv_data['image.size'].get('offset', 0),
)
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated hardcoded method.')
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
try:
met = list(METRICS_CONF['metrics_units']['image'].values())
image_data.append(
self.t_cloudkitty.format_item(
image,
met[0]['unit'],
image_size_mb,
))
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
image_data.append(
self.t_cloudkitty.format_item(
image,
self.units_mappings['image'],
image_size_mb,
))
if not image_data:
raise collector.NoDataCollected(self.collector_name, 'image')
return self.t_cloudkitty.format_service('image', image_data)
def get_radosgw_usage(self,
start,
end=None,
project_id=None,
q_filter=None):
active_rgw_stats = self.resources_stats('radosgw.objects.size', start,
end, project_id, q_filter)
rgw_data = []
for rgw_stats in active_rgw_stats:
rgw_id = rgw_stats.groupby['resource_id']
if not self._cacher.has_resource_detail('radosgw.usage', rgw_id):
raw_resource = self._conn.resources.get(rgw_id)
rgw = self.t_ceilometer.strip_resource_data(
'radosgw.usage', raw_resource)
self._cacher.add_resource_detail('radosgw.usage', rgw_id, rgw)
rgw = self._cacher.get_resource_detail('radosgw.usage', rgw_id)
# Unit conversion
try:
conv_data = METRICS_CONF['metrics_units']['radosgw.usage']
rgw_size = ck_utils.convert_unit(
decimal.Decimal(rgw_stats.max),
conv_data['radosgw.object.size'].get('factor', 1),
conv_data['radosgw.object.size'].get('offset', 0),
)
rgw_data.append(
self.t_cloudkitty.format_item(
rgw,
conv_data['rados.objects.size']['unit'],
rgw_size,
))
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated hardcoded method.')
rgw_size = decimal.Decimal(rgw_stats.max) / units.Gi
rgw_data.append(
self.t_cloudkitty.format_item(
rgw,
self.units_mappings['radosgw.usage'],
rgw_size,
))
if not rgw_data:
raise collector.NoDataCollected(self.collector_name,
'radosgw.usage')
return self.t_cloudkitty.format_service('radosgw.usage', rgw_data)
def _expand_metrics(self, resource, resource_id,
mappings, start, end, resource_type):
for mapping in mappings:
name, statistics = list(mapping.items())[0]
qty = self._get_resource_qty(
name,
start,
end,
resource_id,
statistics,
)
conv_data = METRICS_CONF['metrics'][resource_type].get(name)
if conv_data:
resource[name] = ck_utils.convert_unit(
qty,
conv_data.get('factor', 1),
conv_data.get('offset', 0),
)
def _format_data(self, metconf, data, resources_info=None):
"""Formats Monasca data to CK data.
Returns metadata, groupby and qty
"""
groupby = data['dimensions']
resource_key = metconf['extra_args']['resource_key']
metadata = dict()
if resources_info:
resource = resources_info[groupby[resource_key]]
for i in metconf['metadata']:
metadata[i] = resource.get(i, '')
qty = data['statistics'][0][1]
converted_qty = ck_utils.convert_unit(
qty, metconf['factor'], metconf['offset'])
mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
return metadata, groupby, mutated_qty
def _format_data(self, metconf, data, resources_info=None):
"""Formats gnocchi data to CK data.
Returns metadata, groupby and qty
"""
groupby = data['group']
# if resource info is provided, add additional
# metadata as defined in the conf
metadata = dict()
if resources_info is not None:
resource = resources_info[
groupby[metconf['extra_args']['resource_key']]]
for i in metconf['metadata']:
metadata[i] = resource.get(i, '')
qty = data['measures']['measures']['aggregated'][0][2]
converted_qty = ck_utils.convert_unit(
qty, metconf['factor'], metconf['offset'])
mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
return metadata, groupby, mutated_qty
def _format_data(self, metconf, data, resources_info=None):
"""Formats Monasca data to CK data.
Returns metadata, groupby and qty
"""
groupby = data['dimensions']
resource_key = metconf['extra_args']['resource_key']
metadata = dict()
if resources_info:
resource = resources_info[groupby[resource_key]]
for i in metconf['metadata']:
metadata[i] = resource.get(i, '')
qty = data['statistics'][0][1]
converted_qty = ck_utils.convert_unit(qty, metconf['factor'],
metconf['offset'])
mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
return metadata, groupby, mutated_qty
def _format_data(self, metric_name, scope_key, scope_id, start, end, data):
"""Formats Prometheus data format to Cloudkitty data format.
Returns metadata, groupby, qty
"""
metadata = {}
for meta in self.conf[metric_name]['metadata']:
metadata[meta] = data['metric'][meta]
groupby = {scope_key: scope_id}
for meta in self.conf[metric_name]['groupby']:
groupby[meta] = data['metric'].get(meta, '')
with localcontext() as ctx:
ctx.prec = 9
ctx.rounding = ROUND_HALF_UP
qty = ck_utils.convert_unit(
+Decimal(data['value'][1]),
self.conf[metric_name]['factor'],
self.conf[metric_name]['offset'],
)
return metadata, groupby, qty
def _format_data(self, metconf, data, resources_info=None):
"""Formats gnocchi data to CK data.
Returns metadata, groupby and qty
"""
groupby = data['group']
# if resource info is provided, add additional
# metadata as defined in the conf
metadata = dict()
if resources_info is not None:
resource_key = metconf['extra_args']['resource_key']
resource_id = groupby[resource_key]
try:
resource = resources_info[resource_id]
except KeyError:
raise AssociatedResourceNotFound(resource_key, resource_id)
for i in metconf['metadata']:
metadata[i] = resource.get(i, '')
qty = data['measures']['measures']['aggregated'][0][2]
converted_qty = ck_utils.convert_unit(
qty, metconf['factor'], metconf['offset'])
mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'])
return metadata, groupby, mutated_qty
def test_arg_types(self):
"""Test function with several arg combinations of different types"""
for fac, off in itertools.product(self.possible_args, repeat=2):
factor = fac if fac else 1
offset = off if off else 0
ck_utils.convert_unit(10, factor, offset)
def test_str_float_float(self):
result = ck_utils.convert_unit('1/2', 0.5, 0.5)
self.assertEqual(result, decimal.Decimal(0.5 * 0.5 + 0.5))
def test_arg_types(self):
"""Test function with several arg combinations of different types"""
for fac, off in itertools.product(self.possible_args, repeat=2):
factor = fac if fac else 1
offset = off if off else 0
ck_utils.convert_unit(10, factor, offset)
def resource_info(self,
resource_name,
start,
end,
project_id,
q_filter=None):
try:
tmp = METRICS_CONF['metrics_units'][resource_name]
qty = list(tmp.keys())[0]
unit = list(tmp.values())[0]['unit']
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
qty, unit = self.units_mappings.get(
resource_name,
self.default_unit,
)
resources = self.get_resources(resource_name,
start,
end,
project_id=project_id,
q_filter=q_filter)
formated_resources = list()
for resource in resources:
resource_data = self.t_gnocchi.strip_resource_data(
resource_name, resource)
try:
mappings = METRICS_CONF['services_metrics'][resource_name]
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated oslo config method.')
mappings = self.metrics_mappings[resource_name]
self._expand_metrics([resource_data], mappings, start, end)
resource_data.pop('metrics', None)
# Unit conversion
try:
conv_data = METRICS_CONF['metrics_units'][resource_name][qty]
if isinstance(qty, str):
resource_data[qty] = ck_utils.convert_unit(
resource_data[qty],
conv_data.get('factor', '1'),
conv_data.get('offset', '0'),
)
# NOTE(mc): deprecated except part kept for backward compatibility.
except KeyError:
LOG.warning('Error when trying to use yaml metrology conf.')
LOG.warning('Fallback on the deprecated hardcoded method.')
if resource.get('type') == 'instance_network_interface':
resource_data[qty] = (decimal.Decimal(resource_data[qty]) /
units.M)
elif resource.get('type') == 'image':
resource_data[qty] = (decimal.Decimal(resource_data[qty]) /
units.Mi)
elif resource.get('type') == 'ceph_account':
resource_data[qty] = (decimal.Decimal(resource_data[qty]) /
units.Gi)
data = self.t_cloudkitty.format_item(
resource_data, unit,
decimal.Decimal(
qty if isinstance(qty, int) else resource_data[qty]))
# NOTE(sheeprine): Reference to gnocchi resource used by storage
data['resource_id'] = data['desc']['resource_id']
formated_resources.append(data)
return formated_resources
def test_str_float_float(self):
result = ck_utils.convert_unit('1/2', 0.5, 0.5)
self.assertEqual(result, decimal.Decimal(0.5 * 0.5 + 0.5))