def serialize(self):
return {
'timespan': None
if self.timespan is None
else float(utils.timespan_total_seconds(self.timespan)),
'granularity': float(
utils.timespan_total_seconds(self.granularity)),
'points': self.points,
}
def serialize(self):
return {
'timespan': None
if self.timespan is None
else float(utils.timespan_total_seconds(self.timespan)),
'granularity': float(
utils.timespan_total_seconds(self.granularity)),
'points': self.points,
}
def jsonify(self):
"""Return a dict representation with human readable values."""
return {
'timespan': six.text_type(
datetime.timedelta(
seconds=utils.timespan_total_seconds(
self.timespan)))
if self.timespan is not None
else None,
'granularity': six.text_type(
datetime.timedelta(
seconds=utils.timespan_total_seconds(
self.granularity))),
'points': self.points,
}
def jsonify(self):
"""Return a dict representation with human readable values."""
return {
'timespan': six.text_type(
datetime.timedelta(
seconds=utils.timespan_total_seconds(
self.timespan)))
if self.timespan is not None
else None,
'granularity': six.text_type(
datetime.timedelta(
seconds=utils.timespan_total_seconds(
self.granularity))),
'points': self.points,
}
def _list_split_keys(self, metric, aggregation, granularity, version=3):
bucket = self._bucket_name
keys = set()
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
try:
response = self.s3.list_objects_v2(
Bucket=bucket,
Prefix=self._prefix(metric) + '%s_%s' % (
aggregation,
utils.timespan_total_seconds(granularity),
),
**kwargs)
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == "NoSuchKey":
raise storage.MetricDoesNotExist(metric)
raise
for f in response.get('Contents', ()):
try:
meta = f['Key'].split('_')
if (self._version_check(f['Key'], version)):
keys.add(meta[2])
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def __init__(self, name, back_window, definition,
aggregation_methods=None):
self.name = name
self.back_window = back_window
self.definition = []
for d in definition:
if isinstance(d, ArchivePolicyItem):
self.definition.append(d)
elif isinstance(d, dict):
self.definition.append(ArchivePolicyItem(**d))
elif len(d) == 2:
self.definition.append(
ArchivePolicyItem(points=d[0], granularity=d[1]))
else:
raise ValueError(
"Unable to understand policy definition %s" % d)
duplicate_granularities = [
granularity
for granularity, count in collections.Counter(
d.granularity for d in self.definition).items()
if count > 1
]
if duplicate_granularities:
raise ValueError(
"More than one archive policy "
"uses granularity `%s'"
% utils.timespan_total_seconds(duplicate_granularities[0])
)
if aggregation_methods is None:
self.aggregation_methods = self.DEFAULT_AGGREGATION_METHODS
else:
self.aggregation_methods = aggregation_methods
def _object_name(split_key, aggregation, version=3):
name = '%s_%s_%s' % (
aggregation,
utils.timespan_total_seconds(split_key.sampling),
split_key,
)
return name + '_v%s' % version if version else name
def _object_name(split_key, aggregation, version=3):
name = '%s_%s_%s' % (
split_key,
aggregation,
utils.timespan_total_seconds(split_key.sampling),
)
return name + '_v%s' % version if version else name
def __init__(self, name, back_window, definition,
aggregation_methods=None):
self.name = name
self.back_window = back_window
self.definition = []
for d in definition:
if isinstance(d, ArchivePolicyItem):
self.definition.append(d)
elif isinstance(d, dict):
self.definition.append(ArchivePolicyItem(**d))
elif len(d) == 2:
self.definition.append(
ArchivePolicyItem(points=d[0], granularity=d[1]))
else:
raise ValueError(
"Unable to understand policy definition %s" % d)
duplicate_granularities = [
granularity
for granularity, count in collections.Counter(
d.granularity for d in self.definition).items()
if count > 1
]
if duplicate_granularities:
raise ValueError(
"More than one archive policy "
"uses granularity `%s'"
% utils.timespan_total_seconds(duplicate_granularities[0])
)
if aggregation_methods is None:
self.aggregation_methods = self.DEFAULT_AGGREGATION_METHODS
else:
self.aggregation_methods = set(aggregation_methods)
def _list_split_keys(self, metric, aggregation, granularity, version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
keys = set()
granularity = str(utils.timespan_total_seconds(granularity))
for name, value in omaps:
meta = name.split('_')
if (aggregation == meta[3] and granularity == meta[4]
and self._version_check(name, version)):
keys.add(meta[2])
return keys
def _build_metric_path_for_split(self, metric, aggregation,
key, version=3):
path = os.path.join(
self._build_metric_path(metric, aggregation),
str(key)
+ "_"
+ str(utils.timespan_total_seconds(key.sampling)))
return path + '_v%s' % version if version else path
def __init__(self, metric, method, granularity):
self.metric = metric
self.method = method
self.granularity = granularity
super(AggregationDoesNotExist, self).__init__(
"Aggregation method '%s' at granularity '%s' "
"for metric %s does not exist" %
(method, utils.timespan_total_seconds(granularity), metric))
def _build_metric_path_for_split(self, metric, aggregation,
key, version=3):
path = os.path.join(
self._build_metric_path(metric, aggregation),
str(key)
+ "_"
+ str(utils.timespan_total_seconds(key.sampling)))
return path + '_v%s' % version if version else path
def jsonify(self):
return {
"cause": "Aggregation does not exist",
"detail": {
# FIXME(jd) Pecan does not use our JSON renderer for errors
# So we need to convert this
"granularity": utils.timespan_total_seconds(self.granularity),
"aggregation_method": self.method,
},
}
def _aggregated_field_for_split(cls,
aggregation,
key,
version=3,
granularity=None):
path = cls.FIELD_SEP.join([
str(key), aggregation,
str(utils.timespan_total_seconds(granularity or key.sampling))
])
return path + '_v%s' % version if version else path
def retrieve_data(storage_obj, metric, start, stop, window):
"""Retrieves finest-res data available from storage."""
window_seconds = utils.timespan_total_seconds(window)
try:
min_grain = min(
ap.granularity for ap in metric.archive_policy.definition
if (window_seconds %
utils.timespan_total_seconds(ap.granularity) == 0))
except ValueError:
msg = ("No data available that is either full-res or "
"of a granularity that factors into the window size "
"you specified.")
raise aggregates.CustomAggFailure(msg)
data = list(
zip(*storage_obj.get_measures(
metric, start, stop, granularity=min_grain)))
return (min_grain,
pandas.Series(data[2], data[0]) if data else pandas.Series())
def _list_split_keys(self, metric, aggregation, granularity, version=3):
try:
files = os.listdir(self._build_metric_path(metric, aggregation))
except OSError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
keys = set()
granularity = str(utils.timespan_total_seconds(granularity))
for f in files:
meta = f.split("_")
if meta[1] == granularity and self._version_check(f, version):
keys.add(meta[0])
return keys
def _list_split_keys(self, metric, aggregation, granularity, version=3):
container = self._container_name(metric)
try:
headers, files = self.swift.get_container(container,
full_listing=True)
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
keys = set()
granularity = str(utils.timespan_total_seconds(granularity))
for f in files:
try:
meta = f['name'].split('_')
if (aggregation == meta[1] and granularity == meta[2]
and self._version_check(f['name'], version)):
keys.add(meta[0])
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def _list_split_keys_unbatched(self, metric, aggregations, version=3):
bucket = self._bucket_name
keys = {}
for aggregation in aggregations:
keys[aggregation] = set()
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
response = self.s3.list_objects_v2(
Bucket=bucket,
Prefix=self._prefix(metric) + '%s_%s' % (
aggregation.method,
utils.timespan_total_seconds(
aggregation.granularity),
),
**kwargs)
# If response is empty then check that the metric exists
contents = response.get('Contents', ())
if not contents and not self._metric_exists_p(metric, version):
raise storage.MetricDoesNotExist(metric)
for f in contents:
try:
if (self._version_check(f['Key'], version)):
meta = f['Key'].split('_')
keys[aggregation].add(carbonara.SplitKey(
utils.to_timestamp(meta[2]),
sampling=aggregation.granularity))
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def _list_split_keys_unbatched(self, metric, aggregations, version=3):
bucket = self._bucket_name
keys = {}
for aggregation in aggregations:
keys[aggregation] = set()
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
response = self.s3.list_objects_v2(
Bucket=bucket,
Prefix=self._prefix(metric) + '%s_%s' % (
aggregation.method,
utils.timespan_total_seconds(aggregation.granularity),
),
**kwargs)
# If response is empty then check that the metric exists
contents = response.get('Contents', ())
if not contents and not self._metric_exists_p(metric, version):
raise storage.MetricDoesNotExist(metric)
for f in contents:
try:
if (self._version_check(f['Key'], version)):
meta = f['Key'].split('_')
keys[aggregation].add(
carbonara.SplitKey(
utils.to_timestamp(meta[2]),
sampling=aggregation.granularity))
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def __init__(self, metric, granularity):
self.metric = metric
self.granularity = granularity
super(GranularityDoesNotExist, self).__init__(
"Granularity '%s' for metric %s does not exist" %
(utils.timespan_total_seconds(granularity), metric))
def _aggregated_field_for_split(cls, aggregation, key, version=3,
granularity=None):
path = cls.FIELD_SEP.join([
str(key), aggregation,
str(utils.timespan_total_seconds(granularity or key.sampling))])
return path + '_v%s' % version if version else path
def _get_object_name(metric, key, aggregation, version=3):
name = str(
"gnocchi_%s_%s_%s_%s" %
(metric.id, key, aggregation,
utils.timespan_total_seconds(key.sampling)), )
return name + '_v%s' % version if version else name
def get_measures(storage,
metrics_and_aggregations,
operations,
from_timestamp=None,
to_timestamp=None,
granularity=None,
needed_overlap=100.0,
fill=None,
ref_identifier="id"):
"""Get aggregated measures of multiple entities.
:param storage: The storage driver.
:param metrics_and_aggregations: List of metric+agg_method tuple
measured to aggregate.
:param from timestamp: The timestamp to get the measure from.
:param to timestamp: The timestamp to get the measure to.
:param granularity: The granularity to retrieve.
:param fill: The value to use to fill in missing data in series.
"""
references_with_missing_granularity = []
for (metric, aggregation) in metrics_and_aggregations:
if aggregation not in metric.archive_policy.aggregation_methods:
raise gnocchi_storage.AggregationDoesNotExist(metric, aggregation)
if granularity is not None:
for d in metric.archive_policy.definition:
if d.granularity == granularity:
break
else:
references_with_missing_granularity.append(
(getattr(metric, ref_identifier), aggregation))
if references_with_missing_granularity:
raise exceptions.UnAggregableTimeseries(
references_with_missing_granularity,
"granularity '%d' is missing" %
utils.timespan_total_seconds(granularity))
if granularity is None:
granularities = (definition.granularity
for (metric, aggregation) in metrics_and_aggregations
for definition in metric.archive_policy.definition)
granularities_in_common = [
g for g, occurrence in six.iteritems(
collections.Counter(granularities))
if occurrence == len(metrics_and_aggregations)
]
if not granularities_in_common:
raise exceptions.UnAggregableTimeseries(
list((str(getattr(m, ref_identifier)), a)
for (m, a) in metrics_and_aggregations),
'No granularity match')
else:
granularities_in_common = [granularity]
tss = utils.parallel_map(
_get_measures_timeserie,
[(storage, metric, aggregation, ref_identifier, g, from_timestamp,
to_timestamp) for (metric, aggregation) in metrics_and_aggregations
for g in granularities_in_common])
return aggregated(tss, operations, from_timestamp, to_timestamp,
needed_overlap, fill)
def _get_object_name(metric, key, aggregation, version=3):
name = str("gnocchi_%s_%s_%s_%s" % (
metric.id, key, aggregation,
utils.timespan_total_seconds(key.sampling)),
)
return name + '_v%s' % version if version else name
