def _list_split_keys_unbatched(self, metric, aggregations, version=3):
keys = collections.defaultdict(set)
for method, grouped_aggregations in itertools.groupby(
sorted(aggregations, key=ATTRGETTER_METHOD),
ATTRGETTER_METHOD):
try:
files = os.listdir(
self._build_metric_path(metric, method))
except OSError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
raw_keys = list(map(
lambda k: k.split("_"),
filter(
lambda f: self._version_check(f, version),
files)))
if not raw_keys:
continue
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[0])
k_granularities = list(map(utils.to_timespan, zipped[1]))
grouped_aggregations = list(grouped_aggregations)
for timestamp, granularity in six.moves.zip(
k_timestamps, k_granularities):
for agg in grouped_aggregations:
if granularity == agg.granularity:
keys[agg].add(carbonara.SplitKey(
timestamp,
sampling=granularity))
break
return keys
def _list_split_keys(self, metric, aggregations, version=3):
key = self._metric_key(metric)
pipe = self._client.pipeline(transaction=False)
pipe.exists(key)
for aggregation in aggregations:
self._scripts["list_split_keys"](
keys=[key], args=[self._aggregated_field_for_split(
aggregation.method, "*",
version, aggregation.granularity)],
client=pipe,
)
results = pipe.execute()
metric_exists_p = results.pop(0)
if not metric_exists_p:
raise storage.MetricDoesNotExist(metric)
keys = {}
for aggregation, k in six.moves.zip(aggregations, results):
if not k:
keys[aggregation] = set()
continue
timestamps, methods, granularities = list(zip(*k))
timestamps = utils.to_timestamps(timestamps)
granularities = map(utils.to_timespan, granularities)
keys[aggregation] = {
carbonara.SplitKey(timestamp,
sampling=granularity)
for timestamp, granularity
in six.moves.zip(timestamps, granularities)
}
return keys
def _list_split_keys(self, metric, aggregations, version=3):
keys = collections.defaultdict(set)
for method, grouped_aggregations in itertools.groupby(
sorted(aggregations, key=ATTRGETTER_METHOD),
ATTRGETTER_METHOD):
try:
files = os.listdir(self._build_metric_path(metric, method))
except OSError as e:
if e.errno == errno.ENOENT:
raise storage.MetricDoesNotExist(metric)
raise
raw_keys = list(
map(lambda k: k.split("_"),
filter(lambda f: self._version_check(f, version), files)))
if not raw_keys:
continue
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[0])
k_granularities = list(map(utils.to_timespan, zipped[1]))
grouped_aggregations = list(grouped_aggregations)
for timestamp, granularity in six.moves.zip(
k_timestamps, k_granularities):
for agg in grouped_aggregations:
if granularity == agg.granularity:
keys[agg].add(
carbonara.SplitKey(timestamp,
sampling=granularity))
break
return keys
def _list_split_keys(self, metric, aggregations, version=3):
container = self._container_name(metric)
try:
headers, files = self.swift.get_container(container,
full_listing=True)
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
raw_keys = list(
map(lambda k: k.split("_"),
(f['name']
for f in files if self._version_check(f['name'], version)
and not f['name'].startswith('none'))))
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[0])
k_methods = zipped[1]
k_granularities = list(map(utils.to_timespan, zipped[2]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(
carbonara.SplitKey(timestamp, sampling=granularity))
break
return keys
def _list_split_keys_unbatched(self, metric, aggregations, version=3):
container = self._container_name(metric)
try:
headers, files = self.swift.get_container(
container, full_listing=True)
except swclient.ClientException as e:
if e.http_status == 404:
raise storage.MetricDoesNotExist(metric)
raise
raw_keys = list(map(
lambda k: k.split("_"),
(f['name'] for f in files
if self._version_check(f['name'], version)
and not f['name'].startswith('none'))))
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[0])
k_methods = zipped[1]
k_granularities = list(map(utils.to_timespan, zipped[2]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(carbonara.SplitKey(
timestamp,
sampling=granularity))
break
return keys
def _list_split_keys(self, metrics_and_aggregations, version=3):
pipe = self._client.pipeline(transaction=False)
# Keep an ordered list of metrics
metrics = list(metrics_and_aggregations.keys())
for metric in metrics:
key = self._metric_key(metric)
pipe.exists(key)
aggregations = metrics_and_aggregations[metric]
for aggregation in aggregations:
self._scripts["list_split_keys"](
keys=[key],
args=[
self._aggregated_field_for_split(
aggregation.method, "*", version,
aggregation.granularity)
],
client=pipe,
)
results = pipe.execute()
keys = collections.defaultdict(dict)
start = 0
for metric in metrics:
metric_exists_p = results[start]
if not metric_exists_p:
raise storage.MetricDoesNotExist(metric)
aggregations = metrics_and_aggregations[metric]
number_of_aggregations = len(aggregations)
keys_for_aggregations = results[start + 1:start + 1 +
number_of_aggregations]
start += 1 + number_of_aggregations # 1 for metric_exists_p
for aggregation, k in six.moves.zip(aggregations,
keys_for_aggregations):
if not k:
keys[metric][aggregation] = set()
continue
timestamps, methods, granularities = list(zip(*k))
timestamps = utils.to_timestamps(timestamps)
granularities = map(utils.to_timespan, granularities)
keys[metric][aggregation] = {
carbonara.SplitKey(timestamp, sampling=granularity)
for timestamp, granularity in six.moves.zip(
timestamps, granularities)
}
return keys
def _list_split_keys(self, metric, aggregations, version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
raw_keys = [
name.split("_") for name, value in omaps
if self._version_check(name, version)
]
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[2])
k_methods = zipped[3]
k_granularities = list(map(utils.to_timespan, zipped[4]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(
carbonara.SplitKey(timestamp,
sampling=granularity))
break
return keys
def _list_split_keys_unbatched(self, metric, aggregations, version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
raw_keys = [name.split("_")
for name, value in omaps
if self._version_check(name, version)]
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[2])
k_methods = zipped[3]
k_granularities = list(map(utils.to_timespan, zipped[4]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(carbonara.SplitKey(
timestamp,
sampling=granularity))
break
return keys
def _list_split_keys(self, metrics_and_aggregations, version=3):
pipe = self._client.pipeline(transaction=False)
# Keep an ordered list of metrics
metrics = list(metrics_and_aggregations.keys())
for metric in metrics:
key = self._metric_key(metric)
pipe.exists(key)
aggregations = metrics_and_aggregations[metric]
for aggregation in aggregations:
self._scripts["list_split_keys"](
keys=[key], args=[self._aggregated_field_for_split(
aggregation.method, "*",
version, aggregation.granularity)],
client=pipe,
)
results = pipe.execute()
keys = collections.defaultdict(dict)
start = 0
for metric in metrics:
metric_exists_p = results.pop(0)
if not metric_exists_p:
raise storage.MetricDoesNotExist(metric)
aggregations = metrics_and_aggregations[metric]
number_of_aggregations = len(aggregations)
keys_for_aggregations = results[start:number_of_aggregations]
start += number_of_aggregations
for aggregation, k in six.moves.zip(
aggregations, keys_for_aggregations):
if not k:
keys[metric][aggregation] = set()
continue
timestamps, methods, granularities = list(zip(*k))
timestamps = utils.to_timestamps(timestamps)
granularities = map(utils.to_timespan, granularities)
keys[metric][aggregation] = {
carbonara.SplitKey(timestamp,
sampling=granularity)
for timestamp, granularity
in six.moves.zip(timestamps, granularities)
}
return keys
def test_to_timestamp_empty(self):
self.assertEqual([], utils.to_timestamps([]))
