class TestNoneCache(unittest.TestCase):
TEST_METRIC_NAME = "foo.bar"
TEST_METRIC = bg_metric.make_metric_with_defaults(
TEST_METRIC_NAME, bg_metric.MetricMetadata.create())
def setUp(self):
self._accessor = Mock()
self._accessor.TYPE = 'mock'
self._cache = bg_metadata_cache.NoneCache(self._accessor, None)
def test_has_metric_should_always_be_true(self):
self.assertTrue(self._cache.has_metric("foo.bar"))
def test_has_metric_should_not_use_the_accessor(self):
self._cache.has_metric("foo.bar")
self.assert_empty(self._accessor.method_calls)
def test_get_metric_should_always_use_the_accessor(self):
self._accessor.get_metric.return_value = self.TEST_METRIC
self._cache.get_metric(self.TEST_METRIC_NAME)
self._accessor.get_metric.assert_called_once()
self._accessor.get_metric.assert_called_with(self.TEST_METRIC_NAME)
def assert_empty(self, iterable):
self.assertEqual(0, len(iterable))
def make_metric_with_defaults(name, metadata=None, **kwargs):
"""Create a bg_accessor.Metric with specified metadata."""
retention = kwargs.get("retention")
if isinstance(retention, str):
kwargs["retention"] = bg_metric.Retention.from_string(retention)
if metadata:
assert isinstance(metadata, bg_metric.MetricMetadata)
assert not kwargs
else:
metadata = bg_metric.MetricMetadata.create(**kwargs)
return bg_metric.make_metric_with_defaults(name, metadata)
def _document_to_metric(self, document):
metadata = bg_metric.MetricMetadata.from_string_dict(document.config.to_dict())
# TODO: Have a look at dsl doc to avoid parsing strings to dates
# https://github.com/elastic/elasticsearch-dsl-py/blob/master/docs/persistence.rst
return bg_metric.make_metric_with_defaults(
document.name,
metadata,
created_on=ttls.str_to_datetime(document.created_on),
updated_on=ttls.str_to_datetime(document.updated_on),
read_on=ttls.str_to_datetime(document.read_on),
)
def _get_metric_tuples(accessor,
src,
dst,
recursive,
src_retention,
dst_retention,
dry_run=True):
pattern = "%s.**" % src if recursive else src
try:
src_metrics = list_metrics(accessor, pattern)
except TooManyMetrics as e:
log.error("%s; copy aborted" % e)
return
# Prepare retention override
if src_retention:
src_retention = bg_metric.Retention.from_string(src_retention)
if dst_retention:
dst_retention = bg_metric.Retention.from_string(dst_retention)
for src_metric in src_metrics:
src_metadata = src_metric.metadata
if src_retention and src_metric.metadata.retention != src_retention:
src_metric.metadata = bg_metric.MetricMetadata.create(
src_metadata.aggregator, src_retention,
src_metadata.carbon_xfilesfactor)
dst_metric_name = src_metric.name.replace(src, dst, 1)
dst_metric = accessor.get_metric(dst_metric_name)
if dst_metric is None:
log.debug("Metric '%s' was not found and will be created" %
dst_metric_name)
dst_metadata = bg_metric.MetricMetadata.create(
src_metadata.aggregator, dst_retention,
src_metadata.carbon_xfilesfactor)
dst_metric = bg_metric.make_metric_with_defaults(
dst_metric_name, dst_metadata)
if not dry_run:
accessor.create_metric(dst_metric)
elif dst_retention and dst_metric.metadata.retention != dst_retention:
log.debug(
"Metric '%s' was found without '%s' retention and will be updated"
% (dst_metric_name, dst_retention.as_string))
dst_metadata = dst_metric.metadata
dst_metric.metadata = bg_metric.MetricMetadata.create(
dst_metadata.aggregator, dst_retention,
dst_metadata.carbon_xfilesfactor)
if not dry_run:
accessor.update_metric(dst_metric_name,
dst_metric.metadata)
yield (src_metric, dst_metric)
def _document_to_metric(self, document):
metadata = bg_metric.MetricMetadata.from_string_dict(
document.config.to_dict())
# TODO: Have a look at dsl doc to avoid parsing strings to dates
# https://github.com/elastic/elasticsearch-dsl-py/blob/master/docs/persistence.rst
return bg_metric.make_metric_with_defaults(
document.name,
metadata,
created_on=ttls.str_to_datetime(document.created_on),
updated_on=ttls.str_to_datetime(document.updated_on),
read_on=ttls.str_to_datetime(document.read_on),
)
def _get_metric_tuples(
accessor, src, dst, recursive, src_retention, dst_retention, dry_run=True
):
pattern = "%s.**" % src if recursive else src
try:
src_metrics = list_metrics(accessor, pattern)
except TooManyMetrics as e:
log.error("%s; copy aborted" % e)
return
# Prepare retention override
if src_retention:
src_retention = bg_metric.Retention.from_string(src_retention)
if dst_retention:
dst_retention = bg_metric.Retention.from_string(dst_retention)
for src_metric in src_metrics:
src_metadata = src_metric.metadata
if src_retention and src_metric.metadata.retention != src_retention:
src_metric.metadata = bg_metric.MetricMetadata.create(
src_metadata.aggregator, src_retention, src_metadata.carbon_xfilesfactor)
dst_metric_name = src_metric.name.replace(src, dst, 1)
dst_metric = accessor.get_metric(dst_metric_name)
if dst_metric is None:
log.debug(
"Metric '%s' was not found and will be created" % dst_metric_name
)
dst_metadata = bg_metric.MetricMetadata.create(
src_metadata.aggregator, dst_retention, src_metadata.carbon_xfilesfactor)
dst_metric = bg_metric.make_metric_with_defaults(dst_metric_name, dst_metadata)
if not dry_run:
accessor.create_metric(dst_metric)
elif dst_retention and dst_metric.metadata.retention != dst_retention:
log.debug(
"Metric '%s' was found without '%s' retention and will be updated"
% (dst_metric_name, dst_retention.as_string)
)
dst_metadata = dst_metric.metadata
dst_metric.metadata = bg_metric.MetricMetadata.create(
dst_metadata.aggregator, dst_retention, dst_metadata.carbon_xfilesfactor)
if not dry_run:
accessor.update_metric(dst_metric_name, dst_metric.metadata)
yield (src_metric, dst_metric)
def run(self, accessor, opts):
"""Run the command."""
accessor.connect()
metric = accessor.get_metric(opts.metric)
if not metric:
print("Metric '%s' was not found and will be created" % opts.metric)
metadata = bg_metric.MetricMetadata.create(
aggregator=bg_metric.Aggregator.from_config_name(opts.aggregator),
retention=bg_metric.Retention.from_string(opts.retention),
carbon_xfilesfactor=opts.x_files_factor,
)
metric = bg_metric.make_metric_with_defaults(opts.metric, metadata)
accessor.create_metric(metric)
timestamp = int(time.mktime(opts.timestamp.timetuple()))
points = [(timestamp, float(opts.value))] * opts.count
accessor.insert_points_async(metric, points)
def update_metric(self, name, updated_metadata):
"""See bg_accessor.Accessor."""
super(_ElasticSearchAccessor, self).update_metric(name, updated_metadata)
tracing.add_attr_to_trace("metric.name", name)
name = bg_metric.sanitize_metric_name(name)
metric = self.get_metric(name)
if metric is None:
raise InvalidArgumentError("Unknown metric '%s'" % name)
updated_metric = bg_metric.make_metric_with_defaults(
name,
updated_metadata,
created_on=metric.created_on,
updated_on=datetime.datetime.now(),
read_on=metric.read_on,
)
self.create_metric(updated_metric)
def update_metric(self, name, updated_metadata):
"""See bg_accessor.Accessor."""
super(_ElasticSearchAccessor,
self).update_metric(name, updated_metadata)
tracing.add_attr_to_trace("metric.name", name)
name = bg_metric.sanitize_metric_name(name)
metric = self.get_metric(name)
if metric is None:
raise InvalidArgumentError("Unknown metric '%s'" % name)
updated_metric = bg_metric.make_metric_with_defaults(
name,
updated_metadata,
created_on=metric.created_on,
updated_on=datetime.datetime.now(),
read_on=metric.read_on,
)
self.create_metric(updated_metric)
def run(self, accessor, opts):
"""Run the command."""
accessor.connect()
metric = accessor.get_metric(opts.metric)
if not metric:
print("Metric '%s' was not found and will be created" %
opts.metric)
metadata = bg_metric.MetricMetadata.create(
aggregator=bg_metric.Aggregator.from_config_name(
opts.aggregator),
retention=bg_metric.Retention.from_string(opts.retention),
carbon_xfilesfactor=opts.x_files_factor,
)
metric = bg_metric.make_metric_with_defaults(opts.metric, metadata)
accessor.create_metric(metric)
timestamp = int(time.mktime(opts.timestamp.timetuple()))
points = [(timestamp, float(opts.value))] * opts.count
accessor.insert_points_async(metric, points)
def import_whisper(self, path):
if not self._accessor.is_connected:
self._accessor.connect()
name = metric_name_from_wsp(self._opts.root_directory, self._opts.prefix, path)
metadata = self._read_metadata(name, path)
log.debug("%s: %s" % (name, metadata.as_string_dict()))
if not metadata:
return 0
metric = bg_metric.make_metric_with_defaults(name, metadata)
if not self._opts.no_metadata:
self._accessor.create_metric(metric)
ret = 0
if not self._opts.no_data:
points = self._read_points(path)
self._accessor.insert_downsampled_points(metric, points)
ret = len(points)
return ret
def import_whisper(self, path):
if not self._accessor.is_connected:
self._accessor.connect()
name = metric_name_from_wsp(self._opts.root_directory, self._opts.prefix, path)
metadata = self._read_metadata(name, path)
log.debug("%s: %s" % (name, metadata.as_string_dict()))
if not metadata:
return 0
metric = bg_metric.make_metric_with_defaults(name, metadata)
if not self._opts.no_metadata:
self._accessor.create_metric(metric)
ret = 0
if not self._opts.no_data:
points = self._read_points(path)
self._accessor.insert_downsampled_points(metric, points)
ret = len(points)
return ret