Пример #1
0
    def get_index(self, requestContext):
        matches = []

        for root, _, files in walk(settings.WHISPER_DIR):
          root = root.replace(settings.WHISPER_DIR, '')
          for base_name in files:
            if fnmatch.fnmatch(base_name, '*.wsp'):
              match = join(root, base_name).replace('.wsp', '').replace('/', '.').lstrip('.')
              bisect.insort_left(matches, match)

        # unlike 0.9.x, we're going to use os.walk with followlinks
        # since we require Python 2.7 and newer that supports it
        if RRDReader.supported:
          for root, _, files in walk(settings.RRD_DIR, followlinks=True):
            root = root.replace(settings.RRD_DIR, '')
            for base_name in files:
              if fnmatch.fnmatch(base_name, '*.rrd'):
                absolute_path = join(settings.RRD_DIR, root, base_name)
                base_name = splitext(base_name)[0]
                metric_path = join(root, base_name)
                rrd = RRDReader(absolute_path, metric_path)
                for datasource_name in rrd.get_datasources(absolute_path):
                  match = join(metric_path, datasource_name).replace('.rrd', '').replace('/', '.').lstrip('.')
                  if match not in matches:
                    bisect.insort_left(matches, match)

        return matches
Пример #2
0
    def get_index(self, requestContext):
        matches = []

        for root, _, files in walk(settings.WHISPER_DIR):
            root = root.replace(settings.WHISPER_DIR, '')
            for base_name in files:
                if fnmatch.fnmatch(base_name, '*.wsp'):
                    match = join(root, base_name).replace('.wsp', '').replace(
                        '/', '.').lstrip('.')
                    bisect.insort_left(matches, match)

        # unlike 0.9.x, we're going to use os.walk with followlinks
        # since we require Python 2.7 and newer that supports it
        if RRDReader.supported:
            for root, _, files in walk(settings.RRD_DIR, followlinks=True):
                root = root.replace(settings.RRD_DIR, '')
                for base_name in files:
                    if fnmatch.fnmatch(base_name, '*.rrd'):
                        absolute_path = join(settings.RRD_DIR, root, base_name)
                        base_name = splitext(base_name)[0]
                        metric_path = join(root, base_name)
                        rrd = RRDReader(absolute_path, metric_path)
                        for datasource_name in rrd.get_datasources(
                                absolute_path):
                            match = join(metric_path, datasource_name).replace(
                                '.rrd', '').replace('/', '.').lstrip('.')
                            if match not in matches:
                                bisect.insort_left(matches, match)

        return matches
Пример #3
0
    def find_nodes(self, query, reqkey):
        log.info("running blablabla RRd")
        clean_pattern = query.pattern.replace('\\', '')
        pattern_parts = clean_pattern.split('.')

        for root_dir in self.directories:
            for absolute_path in self._find_paths(root_dir, pattern_parts):
                if basename(absolute_path).startswith('.'):
                    continue

                if self.DATASOURCE_DELIMETER in basename(absolute_path):
                    (absolute_path, datasource_pattern) = absolute_path.rsplit(
                        self.DATASOURCE_DELIMETER, 1)
                else:
                    datasource_pattern = None

                relative_path = absolute_path[len(root_dir):].lstrip('/')
                metric_path = fs_to_metric(relative_path)
                real_metric_path = get_real_metric_path(
                    absolute_path, metric_path)

                metric_path_parts = metric_path.split('.')
                for field_index in find_escaped_pattern_fields(query.pattern):
                    metric_path_parts[field_index] = pattern_parts[
                        field_index].replace('\\', '')
                metric_path = '.'.join(metric_path_parts)

                # Now we construct and yield an appropriate Node object
                if isdir(absolute_path):
                    yield BranchNode(metric_path)

                elif isfile(absolute_path):
                    if absolute_path.endswith(
                            '.wsp') and WhisperReader.supported:
                        reader = WhisperReader(absolute_path, real_metric_path)
                        yield LeafNode(metric_path, reader)

                    elif absolute_path.endswith(
                            '.wsp.gz') and GzippedWhisperReader.supported:
                        reader = GzippedWhisperReader(absolute_path,
                                                      real_metric_path)
                        yield LeafNode(metric_path, reader)

                    elif absolute_path.endswith(
                            '.rrd') and RRDReader.supported:
                        if datasource_pattern is None:
                            yield BranchNode(metric_path)

                        else:
                            for datasource_name in RRDReader.get_datasources(
                                    absolute_path):
                                if match_entries([datasource_name],
                                                 datasource_pattern):
                                    reader = RRDReader(absolute_path,
                                                       datasource_name)
                                    yield LeafNode(
                                        metric_path + "." + datasource_name,
                                        reader)
Пример #4
0
    def test_RRDReader_get_intervals(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        reader = RRDReader(self.hostcpu, 'cpu')

        # Intervals are calculated on the actual time so tolerate a 2 second
        # deviation for delays caused between file creation and test.
        for interval in reader.get_intervals():
            self.assertAlmostEqual(interval.start,
                                   self.start_ts - self.points * self.step,
                                   delta=2)
            self.assertAlmostEqual(interval.end, self.start_ts, delta=2)
Пример #5
0
    def test_RRDReader_get_intervals(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        reader = RRDReader(self.hostcpu, 'cpu')

        # Intervals are calculated on the actual time so tolerate a 2 second
        # deviation for delays caused between file creation and test.
        for interval in reader.get_intervals():
            self.assertAlmostEqual(interval.start,
                                   self.start_ts - self.points * self.step,
                                   delta=2)
            self.assertAlmostEqual(interval.end, self.start_ts, delta=2)
Пример #6
0
    def test_RRDReader_fetch(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        # insert some data
        for ts in range(self.start_ts + 60, self.start_ts + 10 * self.step,
                        self.step):
            rrdtool.update(self.hostcpu, '{}:42'.format(ts))

        reader = RRDReader(self.hostcpu, 'cpu')

        (time_info, values) = reader.fetch(self.start_ts + self.step,
                                           self.start_ts + self.step * 2)
        self.assertEqual(list(values), [42.0])
Пример #7
0
    def test_RRDReader_fetch(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        # insert some data
        for ts in range(self.start_ts + 60, self.start_ts + 10 * self.step,
                        self.step):
            rrdtool.update(self.hostcpu, '{}:42'.format(ts))

        reader = RRDReader(self.hostcpu, 'cpu')

        (time_info, values) = reader.fetch(self.start_ts + self.step,
                                           self.start_ts + self.step * 2)
        self.assertEqual(list(values), [42.0])
Пример #8
0
    def find_matches():
        matches = []

        for root, dirs, files in os.walk(settings.WHISPER_DIR):
            root = root.replace(settings.WHISPER_DIR, '')
            for basename in files:
                if fnmatch.fnmatch(basename, '*.wsp'):
                    matches.append(os.path.join(root, basename))

        for root, dirs, files in os.walk(settings.CERES_DIR):
            root = root.replace(settings.CERES_DIR, '')
            for filename in files:
                if filename == '.ceres-node':
                    matches.append(root)

        # unlike 0.9.x, we're going to use os.walk with followlinks
        # since we require Python 2.7 and newer that supports it
        if RRDReader.supported:
            for root, dirs, files in os.walk(settings.RRD_DIR,
                                             followlinks=True):
                root = root.replace(settings.RRD_DIR, '')
                for basename in files:
                    if fnmatch.fnmatch(basename, '*.rrd'):
                        absolute_path = os.path.join(settings.RRD_DIR, root,
                                                     basename)
                        (basename, extension) = os.path.splitext(basename)
                        metric_path = os.path.join(root, basename)
                        rrd = RRDReader(absolute_path, metric_path)
                        for datasource_name in rrd.get_datasources(
                                absolute_path):
                            matches.append(
                                os.path.join(metric_path, datasource_name))

        matches = [
            m.replace('.wsp', '').replace('.rrd', '').replace('/',
                                                              '.').lstrip('.')
            for m in sorted(matches)
        ]
        return matches
Пример #9
0
  def find_matches():
    matches = []

    for root, dirs, files in os.walk(settings.WHISPER_DIR):
      root = root.replace(settings.WHISPER_DIR, '')
      for basename in files:
        if fnmatch.fnmatch(basename, '*.wsp'):
          matches.append(os.path.join(root, basename))

    for root, dirs, files in os.walk(settings.CERES_DIR):
      root = root.replace(settings.CERES_DIR, '')
      for filename in files:
        if filename == '.ceres-node':
          matches.append(root)

    # unlike 0.9.x, we're going to use os.walk with followlinks
    # since we require Python 2.7 and newer that supports it
    if RRDReader.supported:
      for root, dirs, files in os.walk(settings.RRD_DIR, followlinks=True):
        root = root.replace(settings.RRD_DIR, '')
        for basename in files:
          if fnmatch.fnmatch(basename, '*.rrd'):
            absolute_path = os.path.join(settings.RRD_DIR, root, basename)
            (basename,extension) = os.path.splitext(basename)
            metric_path = os.path.join(root, basename)
            rrd = RRDReader(absolute_path, metric_path)
            for datasource_name in rrd.get_datasources(absolute_path):
              matches.append(os.path.join(metric_path, datasource_name))

    matches = [
      m
      .replace('.wsp', '')
      .replace('.rrd', '')
      .replace('/', '.')
      .lstrip('.')
      for m in sorted(matches)
    ]
    return matches
Пример #10
0
  def find_nodes(self, query):
    log.info("running blablabla RRd")
    clean_pattern = query.pattern.replace('\\', '')
    pattern_parts = clean_pattern.split('.')

    for root_dir in self.directories:
      for absolute_path in self._find_paths(root_dir, pattern_parts):
        if basename(absolute_path).startswith('.'):
          continue

        if self.DATASOURCE_DELIMETER in basename(absolute_path):
          (absolute_path, datasource_pattern) = absolute_path.rsplit(self.DATASOURCE_DELIMETER, 1)
        else:
          datasource_pattern = None

        relative_path = absolute_path[ len(root_dir): ].lstrip('/')
        metric_path = fs_to_metric(relative_path)
        real_metric_path = get_real_metric_path(absolute_path, metric_path)

        metric_path_parts = metric_path.split('.')
        for field_index in find_escaped_pattern_fields(query.pattern):
          metric_path_parts[field_index] = pattern_parts[field_index].replace('\\', '')
        metric_path = '.'.join(metric_path_parts)

        # Now we construct and yield an appropriate Node object
        if isdir(absolute_path):
          yield BranchNode(metric_path)

        elif isfile(absolute_path):
          if absolute_path.endswith('.wsp') and WhisperReader.supported:
            reader = WhisperReader(absolute_path, real_metric_path)
            yield LeafNode(metric_path, reader)

          elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
            reader = GzippedWhisperReader(absolute_path, real_metric_path)
            yield LeafNode(metric_path, reader)

          elif absolute_path.endswith('.rrd') and RRDReader.supported:
            if datasource_pattern is None:
              yield BranchNode(metric_path)

            else:
              for datasource_name in RRDReader.get_datasources(absolute_path):
                if match_entries([datasource_name], datasource_pattern):
                  reader = RRDReader(absolute_path, datasource_name)
                  yield LeafNode(metric_path + "." + datasource_name, reader)
Пример #11
0
    def find_nodes(self, query):
        clean_pattern = query.pattern.replace('\\', '')

        # translate query pattern if it is tagged
        tagged = not query.pattern.startswith(
            '_tagged.') and ';' in query.pattern
        if tagged:
            # tagged series are stored in whisper using encoded names, so to retrieve them we need to
            # encode the query pattern using the same scheme used in carbon when they are written.
            encoded_paths = [
                TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=True),
                TaggedSeries.encode(query.pattern, sep=os.sep,
                                    hash_only=False),
            ]

        pattern_parts = clean_pattern.split('.')

        for root_dir in self.directories:
            if tagged:
                relative_paths = []
                for pattern in encoded_paths:
                    entries = [
                        pattern + '.wsp',
                        pattern + '.wsp.gz',
                        pattern + '.rrd',
                    ]
                    for entry in entries:
                        if isfile(join(root_dir, entry)):
                            relative_paths.append(entry)
            else:
                relative_paths = self._find_paths(root_dir, pattern_parts)

            for relative_path in relative_paths:
                if basename(relative_path).startswith('.'):
                    continue

                if self.DATASOURCE_DELIMITER in basename(relative_path):
                    (relative_path, datasource_pattern) = relative_path.rsplit(
                        self.DATASOURCE_DELIMITER, 1)
                else:
                    datasource_pattern = None

                absolute_path = join(root_dir, relative_path)
                metric_path = fs_to_metric(relative_path)
                real_metric_path = get_real_metric_path(
                    absolute_path, metric_path)

                # if we're finding by tag, return the proper metric path
                if tagged:
                    metric_path = query.pattern
                else:
                    metric_path_parts = metric_path.split('.')
                    for field_index in find_escaped_pattern_fields(
                            query.pattern):
                        metric_path_parts[field_index] = pattern_parts[
                            field_index].replace('\\', '')
                    metric_path = '.'.join(metric_path_parts)

                # Now we construct and yield an appropriate Node object
                if isdir(absolute_path):
                    yield BranchNode(metric_path)

                elif absolute_path.endswith(
                        '.wsp') and WhisperReader.supported:
                    reader = WhisperReader(absolute_path, real_metric_path)
                    yield LeafNode(metric_path, reader)

                elif absolute_path.endswith(
                        '.wsp.gz') and GzippedWhisperReader.supported:
                    reader = GzippedWhisperReader(absolute_path,
                                                  real_metric_path)
                    yield LeafNode(metric_path, reader)

                elif absolute_path.endswith('.rrd') and RRDReader.supported:
                    if datasource_pattern is None:
                        yield BranchNode(metric_path)

                    else:
                        for datasource_name in RRDReader.get_datasources(
                                absolute_path):
                            if match_entries([datasource_name],
                                             datasource_pattern):
                                reader = RRDReader(absolute_path,
                                                   datasource_name)
                                yield LeafNode(
                                    metric_path + "." + datasource_name,
                                    reader)
Пример #12
0
    def test_RRDReader_get_datasources(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        datasource = RRDReader.get_datasources(self.hostcpu)
        self.assertEqual(datasource, ['cpu'])
Пример #13
0
    def test_RRDReader_init(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        reader = RRDReader(self.hostcpu, 'hosts.worker1.cpu')
        self.assertIsNotNone(reader)
Пример #14
0
    def test_RRDReader_get_retention(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        retentions = RRDReader.get_retention(self.hostcpu)
        self.assertEqual(retentions, self.points * self.step)
Пример #15
0
 def test_RRDReader_convert_fs_path(self):
     path = RRDReader._convert_fs_path(six.u(self.hostcpu))
     self.assertIsInstance(path, str)
Пример #16
0
    def test_RRDReader_get_retention(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        retentions = RRDReader.get_retention(self.hostcpu)
        self.assertEqual(retentions, self.points * self.step)
Пример #17
0
 def test_RRDReader_convert_fs_path(self):
     path = RRDReader._convert_fs_path(six.u(self.hostcpu))
     self.assertIsInstance(path, str)
Пример #18
0
    def find_nodes(self, query):
        clean_pattern = query.pattern.replace('\\', '')

        # translate query pattern if it is tagged
        tagged = not query.pattern.startswith('_tagged.') and ';' in query.pattern
        if tagged:
          # tagged series are stored in whisper using encoded names, so to retrieve them we need to encode the
          # query pattern using the same scheme used in carbon when they are written.
          clean_pattern = TaggedSeries.encode(query.pattern)

        pattern_parts = clean_pattern.split('.')

        for root_dir in self.directories:
            for absolute_path in self._find_paths(root_dir, pattern_parts):
                if basename(absolute_path).startswith('.'):
                    continue

                if self.DATASOURCE_DELIMITER in basename(absolute_path):
                    (absolute_path, datasource_pattern) = absolute_path.rsplit(
                        self.DATASOURCE_DELIMITER, 1)
                else:
                    datasource_pattern = None

                relative_path = absolute_path[len(root_dir):].lstrip('/')
                metric_path = fs_to_metric(relative_path)
                real_metric_path = get_real_metric_path(
                    absolute_path, metric_path)

                metric_path_parts = metric_path.split('.')
                for field_index in find_escaped_pattern_fields(query.pattern):
                    metric_path_parts[field_index] = pattern_parts[field_index].replace(
                        '\\', '')
                metric_path = '.'.join(metric_path_parts)

                # if we're finding by tag, return the proper metric path
                if tagged:
                  metric_path = query.pattern

                # Now we construct and yield an appropriate Node object
                if isdir(absolute_path):
                    yield BranchNode(metric_path)

                elif isfile(absolute_path):
                    if absolute_path.endswith(
                            '.wsp') and WhisperReader.supported:
                        reader = WhisperReader(absolute_path, real_metric_path)
                        yield LeafNode(metric_path, reader)

                    elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
                        reader = GzippedWhisperReader(
                            absolute_path, real_metric_path)
                        yield LeafNode(metric_path, reader)

                    elif absolute_path.endswith('.rrd') and RRDReader.supported:
                        if datasource_pattern is None:
                            yield BranchNode(metric_path)

                        else:
                            for datasource_name in RRDReader.get_datasources(
                                    absolute_path):
                                if match_entries(
                                        [datasource_name], datasource_pattern):
                                    reader = RRDReader(
                                        absolute_path, datasource_name)
                                    yield LeafNode(metric_path + "." + datasource_name, reader)
Пример #19
0
    def test_RRDReader_get_datasources(self):
        self.create_rrd()
        self.addCleanup(self.wipe_rrd)

        datasource = RRDReader.get_datasources(self.hostcpu)
        self.assertEqual(datasource, ['cpu'])