def _list_split_keys_for_metric(self,
metric,
aggregation,
granularity,
version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
keys = set()
for name, value in omaps:
meta = name.split('_')
if (aggregation == meta[3] and granularity == float(meta[4])
and self._version_check(name, version)):
keys.add(meta[2])
return keys
def _list_split_keys(self, metric, aggregation, granularity, version=3):
bucket = self._bucket_name
keys = set()
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
try:
response = self.s3.list_objects_v2(
Bucket=bucket,
Prefix=self._prefix(metric) + '%s_%s' % (aggregation,
granularity),
**kwargs)
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == "NoSuchKey":
raise storage.MetricDoesNotExist(metric)
raise
for f in response.get('Contents', ()):
try:
meta = f['Key'].split('_')
if (self._version_check(f['Key'], version)):
keys.add(meta[2])
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def _list_split_keys(self, metric, aggregations, version=3):
key = self._metric_key(metric)
pipe = self._client.pipeline(transaction=False)
pipe.exists(key)
for aggregation in aggregations:
self._scripts["list_split_keys"](
keys=[key], args=[self._aggregated_field_for_split(
aggregation.method, "*",
version, aggregation.granularity)],
client=pipe,
)
results = pipe.execute()
metric_exists_p = results.pop(0)
if not metric_exists_p:
raise storage.MetricDoesNotExist(metric)
keys = {}
for aggregation, k in six.moves.zip(aggregations, results):
if not k:
keys[aggregation] = set()
continue
timestamps, methods, granularities = list(zip(*k))
timestamps = utils.to_timestamps(timestamps)
granularities = map(utils.to_timespan, granularities)
keys[aggregation] = {
carbonara.SplitKey(timestamp,
sampling=granularity)
for timestamp, granularity
in six.moves.zip(timestamps, granularities)
}
return keys
def _list_split_keys(self, metric, aggregations, version=3):
keys = collections.defaultdict(set)
for method, grouped_aggregations in itertools.groupby(
sorted(aggregations, key=ATTRGETTER_METHOD),
ATTRGETTER_METHOD):
try:
files = os.listdir(self._build_metric_path(metric, method))
except OSError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
raw_keys = list(
map(lambda k: k.split("_"),
filter(lambda f: self._version_check(f, version), files)))
if not raw_keys:
continue
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[0])
k_granularities = list(map(utils.to_timespan, zipped[1]))
grouped_aggregations = list(grouped_aggregations)
for timestamp, granularity in six.moves.zip(
k_timestamps, k_granularities):
for agg in grouped_aggregations:
if granularity == agg.granularity:
keys[agg].add(
carbonara.SplitKey(timestamp,
sampling=granularity))
break
return keys
def _list_split_keys(self, metric, aggregations, version=3):
container = self._container_name(metric)
try:
headers, files = self.swift.get_container(container,
full_listing=True)
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
raw_keys = list(
map(lambda k: k.split("_"),
(f['name']
for f in files if self._version_check(f['name'], version)
and not f['name'].startswith('none'))))
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[0])
k_methods = zipped[1]
k_granularities = list(map(utils.to_timespan, zipped[2]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(
carbonara.SplitKey(timestamp, sampling=granularity))
break
return keys
def _get_unaggregated_timeserie(self, metric):
try:
headers, contents = self.swift.get_object(
self._container_name(metric), "none")
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
return contents
def _get_unaggregated_timeserie(self, metric):
path = self._build_unaggregated_timeserie_path(metric)
try:
with open(path, 'rb') as f:
return f.read()
except IOError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
def _get_measures(self, metric, key, aggregation, version=3):
redis_key = self._metric_key(metric)
field = self._aggregated_field_for_split(aggregation, key, version)
data = self._client.hget(redis_key, field)
if data is None:
if not self._client.exists(redis_key):
raise storage.MetricDoesNotExist(metric)
raise storage.AggregationDoesNotExist(metric, aggregation)
return data
def _get_measures(self, metric, timestamp_key, aggregation, granularity):
try:
name = self._get_object_name(metric, timestamp_key, aggregation,
granularity)
return self._get_object_content(name)
except rados.ObjectNotFound:
if self._object_exists("gnocchi_%s_container" % metric.id):
raise storage.AggregationDoesNotExist(metric, aggregation)
else:
raise storage.MetricDoesNotExist(metric)
def _get_measures(self, metric, key, aggregation, version=3):
try:
name = self._get_object_name(metric, key, aggregation, version)
return self._get_object_content(name)
except rados.ObjectNotFound:
if self._object_exists(
self._build_unaggregated_timeserie_path(metric, 3)):
raise storage.AggregationDoesNotExist(metric, aggregation)
else:
raise storage.MetricDoesNotExist(metric)
def _get_unaggregated_timeserie(self, metric, version=3):
try:
response = self.s3.get_object(
Bucket=self._bucket_name,
Key=self._build_unaggregated_timeserie_path(metric, version))
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == "NoSuchKey":
raise storage.MetricDoesNotExist(metric)
raise
return response['Body'].read()
def _list_split_keys(self, metric, aggregations, version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
raw_keys = [
name.split("_") for name, value in omaps
if self._version_check(name, version)
]
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[2])
k_methods = zipped[3]
k_granularities = list(map(utils.to_timespan, zipped[4]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(
carbonara.SplitKey(timestamp,
sampling=granularity))
break
return keys
def _list_split_keys(self, metric, aggregation, granularity, version=3):
key = self._metric_key(metric)
if not self._client.exists(key):
raise storage.MetricDoesNotExist(metric)
split_keys = set()
hashes = self._client.hscan_iter(
key, match=self._aggregated_field_for_split(
aggregation, '*', version, granularity))
for f, __ in hashes:
meta = f.decode("utf8").split(self.FIELD_SEP, 1)
split_keys.add(meta[0])
return split_keys
def _get_measures(self, metric, timestamp_key, aggregation, granularity):
path = self._build_metric_path_for_split(metric, aggregation,
timestamp_key, granularity)
try:
with open(path, 'rb') as aggregation_file:
return aggregation_file.read()
except IOError as e:
if e.errno == errno.ENOENT:
if os.path.exists(self._build_metric_dir(metric)):
raise storage.AggregationDoesNotExist(metric, aggregation)
raise storage.MetricDoesNotExist(metric)
raise
def _get_metric_archive(self, metric, aggregation):
"""Retrieve data in the place we used to store TimeSerieArchive."""
path = self._build_metric_archive_path(metric, aggregation)
try:
with open(path, 'rb') as aggregation_file:
return aggregation_file.read()
except IOError as e:
if e.errno == errno.ENOENT:
if os.path.exists(self._build_metric_dir(metric)):
raise storage.AggregationDoesNotExist(metric, aggregation)
raise storage.MetricDoesNotExist(metric)
raise
def _list_split_keys_for_metric(self, metric, aggregation, granularity):
try:
xattrs = self.ioctx.get_xattrs("gnocchi_%s_container" % metric.id)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
keys = []
for xattr, value in xattrs:
_, metric_id, key, agg, g = xattr.split('_', 4)
if aggregation == agg and granularity == float(g):
keys.append(key)
return keys
def _list_split_keys_for_metric(self, metric, aggregation, granularity):
try:
files = os.listdir(self._build_metric_path(metric, aggregation))
except OSError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
keys = []
for f in files:
key, sep, file_granularity = f.partition("_")
if file_granularity == str(granularity):
keys.append(key)
return keys
def _list_split_keys(self, metric, aggregation, granularity, version=3):
try:
files = os.listdir(self._build_metric_path(metric, aggregation))
except OSError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
keys = set()
for f in files:
meta = f.split("_")
if meta[1] == str(granularity) and self._version_check(f, version):
keys.add(meta[0])
return keys
def _get_measures_unbatched(self, metric, key, aggregation, version=3):
path = self._build_metric_path_for_split(metric, aggregation, key,
version)
try:
with open(path, 'rb') as aggregation_file:
return aggregation_file.read()
except IOError as e:
if e.errno == errno.ENOENT:
if os.path.exists(self._build_metric_dir(metric)):
raise storage.AggregationDoesNotExist(
metric, aggregation, key.sampling)
raise storage.MetricDoesNotExist(metric)
raise
def _get_measures(self, metric, keys, aggregation, version=3):
redis_key = self._metric_key(metric)
fields = [
self._aggregated_field_for_split(aggregation, key, version)
for key in keys
]
results = self._client.hmget(redis_key, fields)
for key, data in six.moves.zip(keys, results):
if data is None:
if not self._client.exists(redis_key):
raise storage.MetricDoesNotExist(metric)
raise storage.AggregationDoesNotExist(metric, aggregation,
key.sampling)
return results
def _get_measures(self, metric, key, aggregation, version=3):
try:
headers, contents = self.swift.get_object(
self._container_name(metric), self._object_name(
key, aggregation, version))
except swclient.ClientException as e:
if e.http_status == 404:
try:
self.swift.head_container(self._container_name(metric))
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
raise storage.AggregationDoesNotExist(metric, aggregation)
raise
return contents
def _get_measures(self, metric, key, aggregation, version=3):
try:
response = self.s3.get_object(
Bucket=self._bucket_name,
Key=self._prefix(metric) + self._object_name(
key, aggregation, version))
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == 'NoSuchKey':
try:
response = self.s3.list_objects_v2(
Bucket=self._bucket_name, Prefix=self._prefix(metric))
except botocore.exceptions.ClientError as e:
if e.response['Error'].get('Code') == 'NoSuchKey':
raise storage.MetricDoesNotExist(metric)
raise
raise storage.AggregationDoesNotExist(metric, aggregation)
raise
return response['Body'].read()
def _list_split_keys(self, metrics_and_aggregations, version=3):
pipe = self._client.pipeline(transaction=False)
# Keep an ordered list of metrics
metrics = list(metrics_and_aggregations.keys())
for metric in metrics:
key = self._metric_key(metric)
pipe.exists(key)
aggregations = metrics_and_aggregations[metric]
for aggregation in aggregations:
self._scripts["list_split_keys"](
keys=[key],
args=[
self._aggregated_field_for_split(
aggregation.method, "*", version,
aggregation.granularity)
],
client=pipe,
)
results = pipe.execute()
keys = collections.defaultdict(dict)
start = 0
for metric in metrics:
metric_exists_p = results[start]
if not metric_exists_p:
raise storage.MetricDoesNotExist(metric)
aggregations = metrics_and_aggregations[metric]
number_of_aggregations = len(aggregations)
keys_for_aggregations = results[start + 1:start + 1 +
number_of_aggregations]
start += 1 + number_of_aggregations # 1 for metric_exists_p
for aggregation, k in six.moves.zip(aggregations,
keys_for_aggregations):
if not k:
keys[metric][aggregation] = set()
continue
timestamps, methods, granularities = list(zip(*k))
timestamps = utils.to_timestamps(timestamps)
granularities = map(utils.to_timespan, granularities)
keys[metric][aggregation] = {
carbonara.SplitKey(timestamp, sampling=granularity)
for timestamp, granularity in six.moves.zip(
timestamps, granularities)
}
return keys
def _list_split_keys_for_metric(self, metric, aggregation, granularity):
container = self._container_name(metric)
try:
headers, files = self.swift.get_container(container,
full_listing=True)
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
keys = []
for f in files:
try:
key, agg, g = f['name'].split('_', 2)
except ValueError:
# Might be "none", or any other file. Be resilient.
continue
if aggregation == agg and granularity == float(g):
keys.append(key)
return keys
def _list_split_keys(self, metric, aggregation, granularity, version=3):
container = self._container_name(metric)
try:
headers, files = self.swift.get_container(
container, full_listing=True)
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
keys = set()
for f in files:
try:
meta = f['name'].split('_')
if (aggregation == meta[1] and granularity == float(meta[2])
and self._version_check(f['name'], version)):
keys.add(meta[0])
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def _list_split_keys_unbatched(self, metric, aggregations, version=3):
bucket = self._bucket_name
keys = {}
for aggregation in aggregations:
keys[aggregation] = set()
response = {}
while response.get('IsTruncated', True):
if 'NextContinuationToken' in response:
kwargs = {
'ContinuationToken': response['NextContinuationToken']
}
else:
kwargs = {}
response = self.s3.list_objects_v2(
Bucket=bucket,
Prefix=self._prefix(metric) + '%s_%s' % (
aggregation.method,
utils.timespan_total_seconds(aggregation.granularity),
),
**kwargs)
# If response is empty then check that the metric exists
contents = response.get('Contents', ())
if not contents and not self._metric_exists_p(metric, version):
raise storage.MetricDoesNotExist(metric)
for f in contents:
try:
if (self._version_check(f['Key'], version)):
meta = f['Key'].split('_')
keys[aggregation].add(
carbonara.SplitKey(
utils.to_timestamp(meta[2]),
sampling=aggregation.granularity))
except (ValueError, IndexError):
# Might be "none", or any other file. Be resilient.
continue
return keys
def _get_unaggregated_timeserie(self, metric, version=3):
data = self._client.hget(self._metric_key(metric),
self._unaggregated_field(version))
if data is None:
raise storage.MetricDoesNotExist(metric)
return data
def _get_unaggregated_timeserie(self, metric, version=3):
try:
return self._get_object_content(
self._build_unaggregated_timeserie_path(metric, version))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
def _get_unaggregated_timeserie(self, metric):
try:
return self._get_object_content("gnocchi_%s_none" % metric.id)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
