def _delete_metric(self, metric):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
return
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
return
ops = [self.ioctx.aio_remove(name) for name, _ in omaps]
for op in ops:
op.wait_for_complete_and_cb()
try:
self.ioctx.remove_object(
self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
# It's possible that the object does not exists
pass
def _delete_metric(self, metric):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
return
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
return
ops = [self.ioctx.aio_remove(name) for name, _ in omaps]
for op in ops:
op.wait_for_complete_and_cb()
try:
self.ioctx.remove_object(
self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
# It's possible that the object does not exists
pass
def _list_split_keys_for_metric(self,
metric,
aggregation,
granularity,
version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
keys = set()
for name, value in omaps:
meta = name.split('_')
if (aggregation == meta[3] and granularity == float(meta[4])
and self._version_check(name, version)):
keys.add(meta[2])
return keys
def process_measure_for_metric(self, metric):
sack = self.sack_for_metric(metric.id)
key_prefix = self.MEASURE_PREFIX + "_" + str(metric.id)
measures = []
processed_keys = []
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", key_prefix, -1)
self.ioctx.operate_read_op(op, self.get_sack_name(sack),
flag=self.OMAP_READ_FLAGS)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
# Object has been deleted, so this is just a stalled entry
# in the OMAP listing, ignore
return
for k, v in omaps:
measures.extend(self._unserialize_measures(k, v))
processed_keys.append(k)
yield measures
# Now clean omap
with rados.WriteOpCtx() as op:
# NOTE(sileht): come on Ceph, no return code
# for this operation ?!!
self.ioctx.remove_omap_keys(op, tuple(processed_keys))
self.ioctx.operate_write_op(op, self.get_sack_name(sack),
flags=self.OMAP_WRITE_FLAGS)
def _list_split_keys(self, metric, aggregations, version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
raw_keys = [
name.split("_") for name, value in omaps
if self._version_check(name, version)
]
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[2])
k_methods = zipped[3]
k_granularities = list(map(utils.to_timespan, zipped[4]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(
carbonara.SplitKey(timestamp,
sampling=granularity))
break
return keys
def process_measure_for_metrics(self, metric_ids):
measures = {}
processed_keys = defaultdict(list)
with rados.ReadOpCtx() as op:
for metric_id in metric_ids:
sack = self.sack_for_metric(metric_id)
key_prefix = self.MEASURE_PREFIX + "_" + str(metric_id)
omaps, ret = self.ioctx.get_omap_vals(op, "", key_prefix, -1)
self.ioctx.operate_read_op(op, str(sack),
flag=self.OMAP_READ_FLAGS)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
# Object has been deleted, so this is just a stalled entry
# in the OMAP listing, ignore
continue
m = self._make_measures_array()
for k, v in omaps:
m = numpy.concatenate(
(m, self._unserialize_measures(k, v)))
processed_keys[sack].append(k)
measures[metric_id] = m
yield measures
# Now clean omap
with rados.WriteOpCtx() as op:
for sack, keys in six.iteritems(processed_keys):
# NOTE(sileht): come on Ceph, no return code
# for this operation ?!!
self.ioctx.remove_omap_keys(op, tuple(keys))
self.ioctx.operate_write_op(op, str(sack),
flags=self.OMAP_WRITE_FLAGS)
def _list_split_keys_unbatched(self, metric, aggregations, version=3):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1)
try:
self.ioctx.operate_read_op(
op, self._build_unaggregated_timeserie_path(metric, 3))
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
raise storage.MetricDoesNotExist(metric)
raw_keys = [name.split("_")
for name, value in omaps
if self._version_check(name, version)]
keys = collections.defaultdict(set)
if not raw_keys:
return keys
zipped = list(zip(*raw_keys))
k_timestamps = utils.to_timestamps(zipped[2])
k_methods = zipped[3]
k_granularities = list(map(utils.to_timespan, zipped[4]))
for timestamp, method, granularity in six.moves.zip(
k_timestamps, k_methods, k_granularities):
for aggregation in aggregations:
if (aggregation.method == method
and aggregation.granularity == granularity):
keys[aggregation].add(carbonara.SplitKey(
timestamp,
sampling=granularity))
break
return keys
def _list_keys_to_process(self, sack, prefix="", marker="", limit=-1):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, marker, prefix, limit)
try:
self.ioctx.operate_read_op(
op, str(sack), flag=self.OMAP_READ_FLAGS)
except rados.ObjectNotFound:
# API have still written nothing
return ()
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
return ()
return (k for k, v in omaps)
def _list_keys_to_process(self, sack, prefix="", marker="", limit=-1):
with rados.ReadOpCtx() as op:
omaps, ret = self.ioctx.get_omap_vals(op, marker, prefix, limit)
try:
self.ioctx.operate_read_op(
op, str(sack), flag=self.OMAP_READ_FLAGS)
except rados.ObjectNotFound:
# API have still written nothing
return {}
# NOTE(sileht): after reading the libradospy, I'm
# not sure that ret will have the correct value
# get_omap_vals transforms the C int to python int
# before operate_read_op is called, I dunno if the int
# content is copied during this transformation or if
# this is a pointer to the C int, I think it's copied...
try:
ceph.errno_to_exception(ret)
except rados.ObjectNotFound:
return {}
return dict(omaps)
