def _delete_metric(self, metric): with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1) try: self.ioctx.operate_read_op( op, self._build_unaggregated_timeserie_path(metric, 3)) except rados.ObjectNotFound: return # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: return ops = [self.ioctx.aio_remove(name) for name, _ in omaps] for op in ops: op.wait_for_complete_and_cb() try: self.ioctx.remove_object( self._build_unaggregated_timeserie_path(metric, 3)) except rados.ObjectNotFound: # It's possible that the object does not exists pass
def _list_split_keys_for_metric(self, metric, aggregation, granularity, version=3): with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1) try: self.ioctx.operate_read_op( op, self._build_unaggregated_timeserie_path(metric, 3)) except rados.ObjectNotFound: raise storage.MetricDoesNotExist(metric) # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: raise storage.MetricDoesNotExist(metric) keys = set() for name, value in omaps: meta = name.split('_') if (aggregation == meta[3] and granularity == float(meta[4]) and self._version_check(name, version)): keys.add(meta[2]) return keys
def process_measure_for_metric(self, metric): sack = self.sack_for_metric(metric.id) key_prefix = self.MEASURE_PREFIX + "_" + str(metric.id) measures = [] processed_keys = [] with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, "", key_prefix, -1) self.ioctx.operate_read_op(op, self.get_sack_name(sack), flag=self.OMAP_READ_FLAGS) # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: # Object has been deleted, so this is just a stalled entry # in the OMAP listing, ignore return for k, v in omaps: measures.extend(self._unserialize_measures(k, v)) processed_keys.append(k) yield measures # Now clean omap with rados.WriteOpCtx() as op: # NOTE(sileht): come on Ceph, no return code # for this operation ?!! self.ioctx.remove_omap_keys(op, tuple(processed_keys)) self.ioctx.operate_write_op(op, self.get_sack_name(sack), flags=self.OMAP_WRITE_FLAGS)
def _list_split_keys(self, metric, aggregations, version=3): with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1) try: self.ioctx.operate_read_op( op, self._build_unaggregated_timeserie_path(metric, 3)) except rados.ObjectNotFound: raise storage.MetricDoesNotExist(metric) # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: raise storage.MetricDoesNotExist(metric) raw_keys = [ name.split("_") for name, value in omaps if self._version_check(name, version) ] keys = collections.defaultdict(set) if not raw_keys: return keys zipped = list(zip(*raw_keys)) k_timestamps = utils.to_timestamps(zipped[2]) k_methods = zipped[3] k_granularities = list(map(utils.to_timespan, zipped[4])) for timestamp, method, granularity in six.moves.zip( k_timestamps, k_methods, k_granularities): for aggregation in aggregations: if (aggregation.method == method and aggregation.granularity == granularity): keys[aggregation].add( carbonara.SplitKey(timestamp, sampling=granularity)) break return keys
def process_measure_for_metrics(self, metric_ids): measures = {} processed_keys = defaultdict(list) with rados.ReadOpCtx() as op: for metric_id in metric_ids: sack = self.sack_for_metric(metric_id) key_prefix = self.MEASURE_PREFIX + "_" + str(metric_id) omaps, ret = self.ioctx.get_omap_vals(op, "", key_prefix, -1) self.ioctx.operate_read_op(op, str(sack), flag=self.OMAP_READ_FLAGS) # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: # Object has been deleted, so this is just a stalled entry # in the OMAP listing, ignore continue m = self._make_measures_array() for k, v in omaps: m = numpy.concatenate( (m, self._unserialize_measures(k, v))) processed_keys[sack].append(k) measures[metric_id] = m yield measures # Now clean omap with rados.WriteOpCtx() as op: for sack, keys in six.iteritems(processed_keys): # NOTE(sileht): come on Ceph, no return code # for this operation ?!! self.ioctx.remove_omap_keys(op, tuple(keys)) self.ioctx.operate_write_op(op, str(sack), flags=self.OMAP_WRITE_FLAGS)
def _list_split_keys_unbatched(self, metric, aggregations, version=3): with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, "", "", -1) try: self.ioctx.operate_read_op( op, self._build_unaggregated_timeserie_path(metric, 3)) except rados.ObjectNotFound: raise storage.MetricDoesNotExist(metric) # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: raise storage.MetricDoesNotExist(metric) raw_keys = [name.split("_") for name, value in omaps if self._version_check(name, version)] keys = collections.defaultdict(set) if not raw_keys: return keys zipped = list(zip(*raw_keys)) k_timestamps = utils.to_timestamps(zipped[2]) k_methods = zipped[3] k_granularities = list(map(utils.to_timespan, zipped[4])) for timestamp, method, granularity in six.moves.zip( k_timestamps, k_methods, k_granularities): for aggregation in aggregations: if (aggregation.method == method and aggregation.granularity == granularity): keys[aggregation].add(carbonara.SplitKey( timestamp, sampling=granularity)) break return keys
def _list_keys_to_process(self, sack, prefix="", marker="", limit=-1): with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, marker, prefix, limit) try: self.ioctx.operate_read_op( op, str(sack), flag=self.OMAP_READ_FLAGS) except rados.ObjectNotFound: # API have still written nothing return () # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: return () return (k for k, v in omaps)
def _list_keys_to_process(self, sack, prefix="", marker="", limit=-1): with rados.ReadOpCtx() as op: omaps, ret = self.ioctx.get_omap_vals(op, marker, prefix, limit) try: self.ioctx.operate_read_op( op, str(sack), flag=self.OMAP_READ_FLAGS) except rados.ObjectNotFound: # API have still written nothing return {} # NOTE(sileht): after reading the libradospy, I'm # not sure that ret will have the correct value # get_omap_vals transforms the C int to python int # before operate_read_op is called, I dunno if the int # content is copied during this transformation or if # this is a pointer to the C int, I think it's copied... try: ceph.errno_to_exception(ret) except rados.ObjectNotFound: return {} return dict(omaps)