class TestNoneCache(unittest.TestCase):

    TEST_METRIC_NAME = "foo.bar"
    TEST_METRIC = bg_metric.make_metric_with_defaults(
        TEST_METRIC_NAME, bg_metric.MetricMetadata.create())

    def setUp(self):
        self._accessor = Mock()
        self._accessor.TYPE = 'mock'
        self._cache = bg_metadata_cache.NoneCache(self._accessor, None)

    def test_has_metric_should_always_be_true(self):
        self.assertTrue(self._cache.has_metric("foo.bar"))

    def test_has_metric_should_not_use_the_accessor(self):
        self._cache.has_metric("foo.bar")
        self.assert_empty(self._accessor.method_calls)

    def test_get_metric_should_always_use_the_accessor(self):
        self._accessor.get_metric.return_value = self.TEST_METRIC

        self._cache.get_metric(self.TEST_METRIC_NAME)

        self._accessor.get_metric.assert_called_once()
        self._accessor.get_metric.assert_called_with(self.TEST_METRIC_NAME)

    def assert_empty(self, iterable):
        self.assertEqual(0, len(iterable))
Example #2
0
def make_metric_with_defaults(name, metadata=None, **kwargs):
    """Create a bg_accessor.Metric with specified metadata."""
    retention = kwargs.get("retention")
    if isinstance(retention, str):
        kwargs["retention"] = bg_metric.Retention.from_string(retention)
    if metadata:
        assert isinstance(metadata, bg_metric.MetricMetadata)
        assert not kwargs
    else:
        metadata = bg_metric.MetricMetadata.create(**kwargs)
    return bg_metric.make_metric_with_defaults(name, metadata)
Example #3
0
 def _document_to_metric(self, document):
     metadata = bg_metric.MetricMetadata.from_string_dict(document.config.to_dict())
     # TODO: Have a look at dsl doc to avoid parsing strings to dates
     # https://github.com/elastic/elasticsearch-dsl-py/blob/master/docs/persistence.rst
     return bg_metric.make_metric_with_defaults(
         document.name,
         metadata,
         created_on=ttls.str_to_datetime(document.created_on),
         updated_on=ttls.str_to_datetime(document.updated_on),
         read_on=ttls.str_to_datetime(document.read_on),
     )
Example #4
0
    def _get_metric_tuples(accessor,
                           src,
                           dst,
                           recursive,
                           src_retention,
                           dst_retention,
                           dry_run=True):
        pattern = "%s.**" % src if recursive else src
        try:
            src_metrics = list_metrics(accessor, pattern)
        except TooManyMetrics as e:
            log.error("%s; copy aborted" % e)
            return

        # Prepare retention override
        if src_retention:
            src_retention = bg_metric.Retention.from_string(src_retention)
        if dst_retention:
            dst_retention = bg_metric.Retention.from_string(dst_retention)

        for src_metric in src_metrics:
            src_metadata = src_metric.metadata

            if src_retention and src_metric.metadata.retention != src_retention:
                src_metric.metadata = bg_metric.MetricMetadata.create(
                    src_metadata.aggregator, src_retention,
                    src_metadata.carbon_xfilesfactor)

            dst_metric_name = src_metric.name.replace(src, dst, 1)
            dst_metric = accessor.get_metric(dst_metric_name)

            if dst_metric is None:
                log.debug("Metric '%s' was not found and will be created" %
                          dst_metric_name)
                dst_metadata = bg_metric.MetricMetadata.create(
                    src_metadata.aggregator, dst_retention,
                    src_metadata.carbon_xfilesfactor)
                dst_metric = bg_metric.make_metric_with_defaults(
                    dst_metric_name, dst_metadata)
                if not dry_run:
                    accessor.create_metric(dst_metric)
            elif dst_retention and dst_metric.metadata.retention != dst_retention:
                log.debug(
                    "Metric '%s' was found without '%s' retention and will be updated"
                    % (dst_metric_name, dst_retention.as_string))
                dst_metadata = dst_metric.metadata
                dst_metric.metadata = bg_metric.MetricMetadata.create(
                    dst_metadata.aggregator, dst_retention,
                    dst_metadata.carbon_xfilesfactor)
                if not dry_run:
                    accessor.update_metric(dst_metric_name,
                                           dst_metric.metadata)

            yield (src_metric, dst_metric)
Example #5
0
 def _document_to_metric(self, document):
     metadata = bg_metric.MetricMetadata.from_string_dict(
         document.config.to_dict())
     # TODO: Have a look at dsl doc to avoid parsing strings to dates
     # https://github.com/elastic/elasticsearch-dsl-py/blob/master/docs/persistence.rst
     return bg_metric.make_metric_with_defaults(
         document.name,
         metadata,
         created_on=ttls.str_to_datetime(document.created_on),
         updated_on=ttls.str_to_datetime(document.updated_on),
         read_on=ttls.str_to_datetime(document.read_on),
     )
Example #6
0
    def _get_metric_tuples(
        accessor, src, dst, recursive, src_retention, dst_retention, dry_run=True
    ):
        pattern = "%s.**" % src if recursive else src
        try:
            src_metrics = list_metrics(accessor, pattern)
        except TooManyMetrics as e:
            log.error("%s; copy aborted" % e)
            return

        # Prepare retention override
        if src_retention:
            src_retention = bg_metric.Retention.from_string(src_retention)
        if dst_retention:
            dst_retention = bg_metric.Retention.from_string(dst_retention)

        for src_metric in src_metrics:
            src_metadata = src_metric.metadata

            if src_retention and src_metric.metadata.retention != src_retention:
                src_metric.metadata = bg_metric.MetricMetadata.create(
                        src_metadata.aggregator, src_retention, src_metadata.carbon_xfilesfactor)

            dst_metric_name = src_metric.name.replace(src, dst, 1)
            dst_metric = accessor.get_metric(dst_metric_name)

            if dst_metric is None:
                log.debug(
                    "Metric '%s' was not found and will be created" % dst_metric_name
                )
                dst_metadata = bg_metric.MetricMetadata.create(
                        src_metadata.aggregator, dst_retention, src_metadata.carbon_xfilesfactor)
                dst_metric = bg_metric.make_metric_with_defaults(dst_metric_name, dst_metadata)
                if not dry_run:
                    accessor.create_metric(dst_metric)
            elif dst_retention and dst_metric.metadata.retention != dst_retention:
                log.debug(
                    "Metric '%s' was found without '%s' retention and will be updated"
                    % (dst_metric_name, dst_retention.as_string)
                )
                dst_metadata = dst_metric.metadata
                dst_metric.metadata = bg_metric.MetricMetadata.create(
                        dst_metadata.aggregator, dst_retention, dst_metadata.carbon_xfilesfactor)
                if not dry_run:
                    accessor.update_metric(dst_metric_name, dst_metric.metadata)

            yield (src_metric, dst_metric)
Example #7
0
    def run(self, accessor, opts):
        """Run the command."""
        accessor.connect()

        metric = accessor.get_metric(opts.metric)
        if not metric:
            print("Metric '%s' was not found and will be created" % opts.metric)
            metadata = bg_metric.MetricMetadata.create(
                aggregator=bg_metric.Aggregator.from_config_name(opts.aggregator),
                retention=bg_metric.Retention.from_string(opts.retention),
                carbon_xfilesfactor=opts.x_files_factor,
            )
            metric = bg_metric.make_metric_with_defaults(opts.metric, metadata)
            accessor.create_metric(metric)

        timestamp = int(time.mktime(opts.timestamp.timetuple()))
        points = [(timestamp, float(opts.value))] * opts.count
        accessor.insert_points_async(metric, points)
Example #8
0
    def update_metric(self, name, updated_metadata):
        """See bg_accessor.Accessor."""
        super(_ElasticSearchAccessor, self).update_metric(name, updated_metadata)
        tracing.add_attr_to_trace("metric.name", name)

        name = bg_metric.sanitize_metric_name(name)
        metric = self.get_metric(name)

        if metric is None:
            raise InvalidArgumentError("Unknown metric '%s'" % name)

        updated_metric = bg_metric.make_metric_with_defaults(
            name,
            updated_metadata,
            created_on=metric.created_on,
            updated_on=datetime.datetime.now(),
            read_on=metric.read_on,
        )
        self.create_metric(updated_metric)
Example #9
0
    def update_metric(self, name, updated_metadata):
        """See bg_accessor.Accessor."""
        super(_ElasticSearchAccessor,
              self).update_metric(name, updated_metadata)
        tracing.add_attr_to_trace("metric.name", name)

        name = bg_metric.sanitize_metric_name(name)
        metric = self.get_metric(name)

        if metric is None:
            raise InvalidArgumentError("Unknown metric '%s'" % name)

        updated_metric = bg_metric.make_metric_with_defaults(
            name,
            updated_metadata,
            created_on=metric.created_on,
            updated_on=datetime.datetime.now(),
            read_on=metric.read_on,
        )
        self.create_metric(updated_metric)
Example #10
0
    def run(self, accessor, opts):
        """Run the command."""
        accessor.connect()

        metric = accessor.get_metric(opts.metric)
        if not metric:
            print("Metric '%s' was not found and will be created" %
                  opts.metric)
            metadata = bg_metric.MetricMetadata.create(
                aggregator=bg_metric.Aggregator.from_config_name(
                    opts.aggregator),
                retention=bg_metric.Retention.from_string(opts.retention),
                carbon_xfilesfactor=opts.x_files_factor,
            )
            metric = bg_metric.make_metric_with_defaults(opts.metric, metadata)
            accessor.create_metric(metric)

        timestamp = int(time.mktime(opts.timestamp.timetuple()))
        points = [(timestamp, float(opts.value))] * opts.count
        accessor.insert_points_async(metric, points)
Example #11
0
    def import_whisper(self, path):
        if not self._accessor.is_connected:
            self._accessor.connect()

        name = metric_name_from_wsp(self._opts.root_directory, self._opts.prefix, path)
        metadata = self._read_metadata(name, path)
        log.debug("%s: %s" % (name, metadata.as_string_dict()))

        if not metadata:
            return 0

        metric = bg_metric.make_metric_with_defaults(name, metadata)
        if not self._opts.no_metadata:
            self._accessor.create_metric(metric)

        ret = 0
        if not self._opts.no_data:
            points = self._read_points(path)
            self._accessor.insert_downsampled_points(metric, points)
            ret = len(points)

        return ret
Example #12
0
    def import_whisper(self, path):
        if not self._accessor.is_connected:
            self._accessor.connect()

        name = metric_name_from_wsp(self._opts.root_directory, self._opts.prefix, path)
        metadata = self._read_metadata(name, path)
        log.debug("%s: %s" % (name, metadata.as_string_dict()))

        if not metadata:
            return 0

        metric = bg_metric.make_metric_with_defaults(name, metadata)
        if not self._opts.no_metadata:
            self._accessor.create_metric(metric)

        ret = 0
        if not self._opts.no_data:
            points = self._read_points(path)
            self._accessor.insert_downsampled_points(metric, points)
            ret = len(points)

        return ret