def test_taggedseries(self): # test path with tags parsed = TaggedSeries.parse('test.a;hello=tiger;blah=blah') self.assertIsInstance(parsed, TaggedSeries) self.assertEqual(parsed.metric, 'test.a') self.assertEqual(parsed.tags, {'blah': 'blah', 'hello': 'tiger', 'name': 'test.a'}) # test formatting self.assertEqual(parsed.path, 'test.a;blah=blah;hello=tiger') # test encoding self.assertEqual(TaggedSeries.encode(parsed.path), '_tagged.2b0.2af.test-a;blah=blah;hello=tiger') # test path without tags parsed = TaggedSeries.parse('test.a') self.assertIsInstance(parsed, TaggedSeries) self.assertEqual(parsed.metric, 'test.a') self.assertEqual(parsed.tags, {'name': 'test.a'}) # test formatting self.assertEqual(parsed.path, 'test.a') # test encoding self.assertEqual(TaggedSeries.encode(parsed.path), 'test.a') # test parsing openmetrics parsed = TaggedSeries.parse(r'test.a{hello="tiger",blah="bla\"h"}') self.assertIsInstance(parsed, TaggedSeries) self.assertEqual(parsed.metric, 'test.a') self.assertEqual(parsed.tags, {'blah': 'bla"h', 'hello': 'tiger', 'name': 'test.a'})
def find_nodes(self, query): # translate query pattern if it is tagged tagged = not query.pattern.startswith('_tagged.') and ';' in query.pattern if tagged: # tagged series are stored in ceres using encoded names, so to retrieve them we need to # encode the query pattern using the same scheme used in carbon when they are written. variants = [ TaggedSeries.encode(query.pattern, hash_only=True), TaggedSeries.encode(query.pattern, hash_only=False), ] else: variants = extract_variants(query.pattern) for variant in variants: for fs_path in glob(self.tree.getFilesystemPath(variant)): metric_path = self.tree.getNodePath(fs_path) if CeresNode.isNodeDir(fs_path): ceres_node = self.tree.getNode(metric_path) if ceres_node.hasDataForInterval(query.startTime, query.endTime): real_metric_path = get_real_metric_path(fs_path, metric_path) reader = CeresReader(ceres_node, real_metric_path) # if we're finding by tag, return the proper metric path if tagged: metric_path = query.pattern yield LeafNode(metric_path, reader) elif os.path.isdir(fs_path): yield BranchNode(metric_path)
def test_taggedseries(self): # test path with tags parsed = TaggedSeries.parse('test.a;hello=tiger;blah=blah') self.assertIsInstance(parsed, TaggedSeries) self.assertEqual(parsed.metric, 'test.a') self.assertEqual(parsed.tags, {'blah': 'blah', 'hello': 'tiger', 'name': 'test.a'}) # test formatting self.assertEqual(parsed.path, 'test.a;blah=blah;hello=tiger') # test encoding self.assertEqual(TaggedSeries.encode(parsed.path), '_tagged.2b0.2af.test-a;blah=blah;hello=tiger') # test path without tags parsed = TaggedSeries.parse('test.a') self.assertIsInstance(parsed, TaggedSeries) self.assertEqual(parsed.metric, 'test.a') self.assertEqual(parsed.tags, {'name': 'test.a'}) # test formatting self.assertEqual(parsed.path, 'test.a') # test encoding self.assertEqual(TaggedSeries.encode(parsed.path), 'test.a') # test parsing openmetrics parsed = TaggedSeries.parse(r'test.a{hello="tiger",blah="bla\"h"}') self.assertIsInstance(parsed, TaggedSeries) self.assertEqual(parsed.metric, 'test.a') self.assertEqual(parsed.tags, {'blah': 'bla"h', 'hello': 'tiger', 'name': 'test.a'})
def find_nodes(self, query): # translate query pattern if it is tagged tagged = not query.pattern.startswith( '_tagged.') and ';' in query.pattern if tagged: # tagged series are stored in ceres using encoded names, so to retrieve them we need to encode the # query pattern using the same scheme used in carbon when they are written. variants = [TaggedSeries.encode(query.pattern)] else: variants = extract_variants(query.pattern) for variant in variants: for fs_path in glob(self.tree.getFilesystemPath(variant)): metric_path = self.tree.getNodePath(fs_path) if CeresNode.isNodeDir(fs_path): ceres_node = self.tree.getNode(metric_path) if ceres_node.hasDataForInterval(query.startTime, query.endTime): real_metric_path = get_real_metric_path( fs_path, metric_path) reader = CeresReader(ceres_node, real_metric_path) # if we're finding by tag, return the proper metric path if tagged: metric_path = query.pattern yield LeafNode(metric_path, reader) elif os.path.isdir(fs_path): yield BranchNode(metric_path)
def find_nodes(self, query): clean_pattern = query.pattern.replace('\\', '') # translate query pattern if it is tagged tagged = not query.pattern.startswith('_tagged.') and ';' in query.pattern if tagged: # tagged series are stored in whisper using encoded names, so to retrieve them we need to encode the # query pattern using the same scheme used in carbon when they are written. clean_pattern = TaggedSeries.encode(query.pattern) pattern_parts = clean_pattern.split('.') for root_dir in self.directories: for absolute_path in self._find_paths(root_dir, pattern_parts): if basename(absolute_path).startswith('.'): continue if self.DATASOURCE_DELIMITER in basename(absolute_path): (absolute_path, datasource_pattern) = absolute_path.rsplit( self.DATASOURCE_DELIMITER, 1) else: datasource_pattern = None relative_path = absolute_path[len(root_dir):].lstrip('/') metric_path = fs_to_metric(relative_path) real_metric_path = get_real_metric_path( absolute_path, metric_path) metric_path_parts = metric_path.split('.') for field_index in find_escaped_pattern_fields(query.pattern): metric_path_parts[field_index] = pattern_parts[field_index].replace( '\\', '') metric_path = '.'.join(metric_path_parts) # if we're finding by tag, return the proper metric path if tagged: metric_path = query.pattern # Now we construct and yield an appropriate Node object if isdir(absolute_path): yield BranchNode(metric_path) elif isfile(absolute_path): if absolute_path.endswith( '.wsp') and WhisperReader.supported: reader = WhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported: reader = GzippedWhisperReader( absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith('.rrd') and RRDReader.supported: if datasource_pattern is None: yield BranchNode(metric_path) else: for datasource_name in RRDReader.get_datasources( absolute_path): if match_entries( [datasource_name], datasource_pattern): reader = RRDReader( absolute_path, datasource_name) yield LeafNode(metric_path + "." + datasource_name, reader)
def find_nodes(self, query): clean_pattern = query.pattern.replace('\\', '') # translate query pattern if it is tagged tagged = not query.pattern.startswith( '_tagged.') and ';' in query.pattern if tagged: # tagged series are stored in whisper using encoded names, so to retrieve them we need to # encode the query pattern using the same scheme used in carbon when they are written. encoded_paths = [ TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=True), TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=False), ] pattern_parts = clean_pattern.split('.') for root_dir in self.directories: if tagged: relative_paths = [] for pattern in encoded_paths: entries = [ pattern + '.wsp', pattern + '.wsp.gz', pattern + '.rrd', ] for entry in entries: if isfile(join(root_dir, entry)): relative_paths.append(entry) else: relative_paths = self._find_paths(root_dir, pattern_parts) for relative_path in relative_paths: if basename(relative_path).startswith('.'): continue if self.DATASOURCE_DELIMITER in basename(relative_path): (relative_path, datasource_pattern) = relative_path.rsplit( self.DATASOURCE_DELIMITER, 1) else: datasource_pattern = None absolute_path = join(root_dir, relative_path) metric_path = fs_to_metric(relative_path) real_metric_path = get_real_metric_path( absolute_path, metric_path) # if we're finding by tag, return the proper metric path if tagged: metric_path = query.pattern else: metric_path_parts = metric_path.split('.') for field_index in find_escaped_pattern_fields( query.pattern): metric_path_parts[field_index] = pattern_parts[ field_index].replace('\\', '') metric_path = '.'.join(metric_path_parts) # Now we construct and yield an appropriate Node object if isdir(absolute_path): yield BranchNode(metric_path) elif absolute_path.endswith( '.wsp') and WhisperReader.supported: reader = WhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith( '.wsp.gz') and GzippedWhisperReader.supported: reader = GzippedWhisperReader(absolute_path, real_metric_path) yield LeafNode(metric_path, reader) elif absolute_path.endswith('.rrd') and RRDReader.supported: if datasource_pattern is None: yield BranchNode(metric_path) else: for datasource_name in RRDReader.get_datasources( absolute_path): if match_entries([datasource_name], datasource_pattern): reader = RRDReader(absolute_path, datasource_name) yield LeafNode( metric_path + "." + datasource_name, reader)