def _expand_metrics(self, resource, resource_id, mappings, start, end, resource_type): for mapping in mappings: name, statistics = list(mapping.items())[0] qty = self._get_resource_qty( name, start, end, resource_id, statistics, ) try: conv_data = METRICS_CONF['metrics_units'][resource_type] conv_data = conv_data.get(name) if conv_data: resource[name] = ck_utils.convert_unit( qty, conv_data.get('factor', 1), conv_data.get('offset', 0), ) # NOTE(mc): deprecated except part kept for backward compatibility. except KeyError: LOG.warning( 'Error when trying to use yaml metrology conf.\n' 'Fallback on the deprecated hardcoded dict method.') names = ['network.outgoing.bytes', 'network.incoming.bytes'] if name in names: qty = qty / units.M elif 'image.' in name: qty = qty / units.Mi resource[name] = qty
def _format_data(self, metric_name, scope_key, scope_id, start, end, data): """Formats Prometheus data format to Cloudkitty data format. Returns metadata, groupby, qty """ metadata = {} for meta in self.conf[metric_name]['metadata']: metadata[meta] = data['metric'].get(meta, '') groupby = {scope_key: scope_id} for meta in self.conf[metric_name]['groupby']: groupby[meta] = data['metric'].get(meta, '') with localcontext() as ctx: ctx.prec = 9 ctx.rounding = ROUND_HALF_UP qty = ck_utils.convert_unit( +Decimal(data['value'][1]), self.conf[metric_name]['factor'], self.conf[metric_name]['offset'], ) qty = ck_utils.mutate(qty, self.conf[metric_name]['mutate']) return metadata, groupby, qty
def _get_network_bw(self, direction, start, end=None, project_id=None, q_filter=None): if direction == 'in': resource_type = 'network.incoming.bytes' else: direction = 'out' resource_type = 'network.outgoing.bytes' active_tap_stats = self.resources_stats(resource_type, start, end, project_id, q_filter) bw_data = [] for tap_stat in active_tap_stats: tap_id = tap_stat.groupby['resource_id'] if not self._cacher.has_resource_detail('network.tap', tap_id): raw_resource = self._conn.resources.get(tap_id) tap = self.t_ceilometer.strip_resource_data( 'network.tap', raw_resource) self._cacher.add_resource_detail('network.tap', tap_id, tap) tap = self._cacher.get_resource_detail('network.tap', tap_id) # Unit conversion try: conv = METRICS_CONF['metrics_units']['network.bw.' + direction] tap_bw_mb = ck_utils.convert_unit( decimal.Decimal(tap_stat.max), conv[resource_type].get('factor', 1), conv[resource_type].get('offset', 0), ) except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated hardcoded method.') tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M try: met = METRICS_CONF['metrics_units']['network.bw.' + direction] bw_data.append( self.t_cloudkitty.format_item( tap, list(met.values())[0]['unit'], tap_bw_mb, )) # NOTE(mc): deprecated except part kept for backward compatibility. except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated oslo config method.') bw_data.append( self.t_cloudkitty.format_item( tap, self.units_mappings['network.bw.' + direction], tap_bw_mb, )) ck_res_name = 'network.bw.{}'.format(direction) if not bw_data: raise collector.NoDataCollected(self.collector_name, ck_res_name) return self.t_cloudkitty.format_service(ck_res_name, bw_data)
def resource_info(self, resource_name, start, end, project_id, q_filter=None): met = self.conf['metrics'][resource_name] unit = met['unit'] qty = 1 if met.get('countable_unit') else met['resource'] resources = self.get_resources( resource_name, start, end, project_id=project_id, q_filter=q_filter, ) formated_resources = list() for resource in resources: resource_data = self.t_gnocchi.strip_resource_data( resource_name, resource) mapp = self.conf['metrics'][resource_name]['aggregation_method'] self._expand_metrics( [resource_data], mapp, start, end, resource_name, ) resource_data.pop('metrics', None) # Unit conversion if isinstance(qty, str): resource_data[resource_name] = ck_utils.convert_unit( resource_data[resource_name], self.conf['metrics'][resource_name].get('factor', 1), self.conf['metrics'][resource_name].get('offset', 0), ) val = qty if isinstance(qty, int) else resource_data[resource_name] data = self.t_cloudkitty.format_item( resource_data, unit, decimal.Decimal(val) ) # NOTE(sheeprine): Reference to gnocchi resource used by storage data['resource_id'] = data['desc']['resource_id'] formated_resources.append(data) return formated_resources
def get_image(self, start, end=None, project_id=None, q_filter=None): active_image_stats = self.resources_stats('image.size', start, end, project_id, q_filter) image_data = [] for image_stats in active_image_stats: image_id = image_stats.groupby['resource_id'] if not self._cacher.has_resource_detail('image', image_id): raw_resource = self._conn.resources.get(image_id) image = self.t_ceilometer.strip_resource_data( 'image', raw_resource) self._cacher.add_resource_detail('image', image_id, image) image = self._cacher.get_resource_detail('image', image_id) # Unit conversion try: conv_data = METRICS_CONF['metrics_units']['image'] image_size_mb = ck_utils.convert_unit( decimal.Decimal(image_stats.max), conv_data['image.size'].get('factor', 1), conv_data['image.size'].get('offset', 0), ) except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated hardcoded method.') image_size_mb = decimal.Decimal(image_stats.max) / units.Mi try: met = list(METRICS_CONF['metrics_units']['image'].values()) image_data.append( self.t_cloudkitty.format_item( image, met[0]['unit'], image_size_mb, )) # NOTE(mc): deprecated except part kept for backward compatibility. except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated oslo config method.') image_data.append( self.t_cloudkitty.format_item( image, self.units_mappings['image'], image_size_mb, )) if not image_data: raise collector.NoDataCollected(self.collector_name, 'image') return self.t_cloudkitty.format_service('image', image_data)
def get_radosgw_usage(self, start, end=None, project_id=None, q_filter=None): active_rgw_stats = self.resources_stats('radosgw.objects.size', start, end, project_id, q_filter) rgw_data = [] for rgw_stats in active_rgw_stats: rgw_id = rgw_stats.groupby['resource_id'] if not self._cacher.has_resource_detail('radosgw.usage', rgw_id): raw_resource = self._conn.resources.get(rgw_id) rgw = self.t_ceilometer.strip_resource_data( 'radosgw.usage', raw_resource) self._cacher.add_resource_detail('radosgw.usage', rgw_id, rgw) rgw = self._cacher.get_resource_detail('radosgw.usage', rgw_id) # Unit conversion try: conv_data = METRICS_CONF['metrics_units']['radosgw.usage'] rgw_size = ck_utils.convert_unit( decimal.Decimal(rgw_stats.max), conv_data['radosgw.object.size'].get('factor', 1), conv_data['radosgw.object.size'].get('offset', 0), ) rgw_data.append( self.t_cloudkitty.format_item( rgw, conv_data['rados.objects.size']['unit'], rgw_size, )) except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated hardcoded method.') rgw_size = decimal.Decimal(rgw_stats.max) / units.Gi rgw_data.append( self.t_cloudkitty.format_item( rgw, self.units_mappings['radosgw.usage'], rgw_size, )) if not rgw_data: raise collector.NoDataCollected(self.collector_name, 'radosgw.usage') return self.t_cloudkitty.format_service('radosgw.usage', rgw_data)
def _expand_metrics(self, resource, resource_id, mappings, start, end, resource_type): for mapping in mappings: name, statistics = list(mapping.items())[0] qty = self._get_resource_qty( name, start, end, resource_id, statistics, ) conv_data = METRICS_CONF['metrics'][resource_type].get(name) if conv_data: resource[name] = ck_utils.convert_unit( qty, conv_data.get('factor', 1), conv_data.get('offset', 0), )
def _format_data(self, metconf, data, resources_info=None): """Formats Monasca data to CK data. Returns metadata, groupby and qty """ groupby = data['dimensions'] resource_key = metconf['extra_args']['resource_key'] metadata = dict() if resources_info: resource = resources_info[groupby[resource_key]] for i in metconf['metadata']: metadata[i] = resource.get(i, '') qty = data['statistics'][0][1] converted_qty = ck_utils.convert_unit( qty, metconf['factor'], metconf['offset']) mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate']) return metadata, groupby, mutated_qty
def _format_data(self, metconf, data, resources_info=None): """Formats gnocchi data to CK data. Returns metadata, groupby and qty """ groupby = data['group'] # if resource info is provided, add additional # metadata as defined in the conf metadata = dict() if resources_info is not None: resource = resources_info[ groupby[metconf['extra_args']['resource_key']]] for i in metconf['metadata']: metadata[i] = resource.get(i, '') qty = data['measures']['measures']['aggregated'][0][2] converted_qty = ck_utils.convert_unit( qty, metconf['factor'], metconf['offset']) mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate']) return metadata, groupby, mutated_qty
def _format_data(self, metconf, data, resources_info=None): """Formats Monasca data to CK data. Returns metadata, groupby and qty """ groupby = data['dimensions'] resource_key = metconf['extra_args']['resource_key'] metadata = dict() if resources_info: resource = resources_info[groupby[resource_key]] for i in metconf['metadata']: metadata[i] = resource.get(i, '') qty = data['statistics'][0][1] converted_qty = ck_utils.convert_unit(qty, metconf['factor'], metconf['offset']) mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate']) return metadata, groupby, mutated_qty
def _format_data(self, metric_name, scope_key, scope_id, start, end, data): """Formats Prometheus data format to Cloudkitty data format. Returns metadata, groupby, qty """ metadata = {} for meta in self.conf[metric_name]['metadata']: metadata[meta] = data['metric'][meta] groupby = {scope_key: scope_id} for meta in self.conf[metric_name]['groupby']: groupby[meta] = data['metric'].get(meta, '') with localcontext() as ctx: ctx.prec = 9 ctx.rounding = ROUND_HALF_UP qty = ck_utils.convert_unit( +Decimal(data['value'][1]), self.conf[metric_name]['factor'], self.conf[metric_name]['offset'], ) return metadata, groupby, qty
def _format_data(self, metconf, data, resources_info=None): """Formats gnocchi data to CK data. Returns metadata, groupby and qty """ groupby = data['group'] # if resource info is provided, add additional # metadata as defined in the conf metadata = dict() if resources_info is not None: resource_key = metconf['extra_args']['resource_key'] resource_id = groupby[resource_key] try: resource = resources_info[resource_id] except KeyError: raise AssociatedResourceNotFound(resource_key, resource_id) for i in metconf['metadata']: metadata[i] = resource.get(i, '') qty = data['measures']['measures']['aggregated'][0][2] converted_qty = ck_utils.convert_unit( qty, metconf['factor'], metconf['offset']) mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate']) return metadata, groupby, mutated_qty
def test_arg_types(self): """Test function with several arg combinations of different types""" for fac, off in itertools.product(self.possible_args, repeat=2): factor = fac if fac else 1 offset = off if off else 0 ck_utils.convert_unit(10, factor, offset)
def test_str_float_float(self): result = ck_utils.convert_unit('1/2', 0.5, 0.5) self.assertEqual(result, decimal.Decimal(0.5 * 0.5 + 0.5))
def resource_info(self, resource_name, start, end, project_id, q_filter=None): try: tmp = METRICS_CONF['metrics_units'][resource_name] qty = list(tmp.keys())[0] unit = list(tmp.values())[0]['unit'] # NOTE(mc): deprecated except part kept for backward compatibility. except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated oslo config method.') qty, unit = self.units_mappings.get( resource_name, self.default_unit, ) resources = self.get_resources(resource_name, start, end, project_id=project_id, q_filter=q_filter) formated_resources = list() for resource in resources: resource_data = self.t_gnocchi.strip_resource_data( resource_name, resource) try: mappings = METRICS_CONF['services_metrics'][resource_name] # NOTE(mc): deprecated except part kept for backward compatibility. except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated oslo config method.') mappings = self.metrics_mappings[resource_name] self._expand_metrics([resource_data], mappings, start, end) resource_data.pop('metrics', None) # Unit conversion try: conv_data = METRICS_CONF['metrics_units'][resource_name][qty] if isinstance(qty, str): resource_data[qty] = ck_utils.convert_unit( resource_data[qty], conv_data.get('factor', '1'), conv_data.get('offset', '0'), ) # NOTE(mc): deprecated except part kept for backward compatibility. except KeyError: LOG.warning('Error when trying to use yaml metrology conf.') LOG.warning('Fallback on the deprecated hardcoded method.') if resource.get('type') == 'instance_network_interface': resource_data[qty] = (decimal.Decimal(resource_data[qty]) / units.M) elif resource.get('type') == 'image': resource_data[qty] = (decimal.Decimal(resource_data[qty]) / units.Mi) elif resource.get('type') == 'ceph_account': resource_data[qty] = (decimal.Decimal(resource_data[qty]) / units.Gi) data = self.t_cloudkitty.format_item( resource_data, unit, decimal.Decimal( qty if isinstance(qty, int) else resource_data[qty])) # NOTE(sheeprine): Reference to gnocchi resource used by storage data['resource_id'] = data['desc']['resource_id'] formated_resources.append(data) return formated_resources