def test_taggedseries(self):
# test path with tags
parsed = TaggedSeries.parse('test.a;hello=tiger;blah=blah')
self.assertIsInstance(parsed, TaggedSeries)
self.assertEqual(parsed.metric, 'test.a')
self.assertEqual(parsed.tags, {'blah': 'blah', 'hello': 'tiger', 'name': 'test.a'})
# test formatting
self.assertEqual(parsed.path, 'test.a;blah=blah;hello=tiger')
# test encoding
self.assertEqual(TaggedSeries.encode(parsed.path), '_tagged.2b0.2af.test-a;blah=blah;hello=tiger')
# test path without tags
parsed = TaggedSeries.parse('test.a')
self.assertIsInstance(parsed, TaggedSeries)
self.assertEqual(parsed.metric, 'test.a')
self.assertEqual(parsed.tags, {'name': 'test.a'})
# test formatting
self.assertEqual(parsed.path, 'test.a')
# test encoding
self.assertEqual(TaggedSeries.encode(parsed.path), 'test.a')
# test parsing openmetrics
parsed = TaggedSeries.parse(r'test.a{hello="tiger",blah="bla\"h"}')
self.assertIsInstance(parsed, TaggedSeries)
self.assertEqual(parsed.metric, 'test.a')
self.assertEqual(parsed.tags, {'blah': 'bla"h', 'hello': 'tiger', 'name': 'test.a'})
def find_nodes(self, query):
# translate query pattern if it is tagged
tagged = not query.pattern.startswith('_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in ceres using encoded names, so to retrieve them we need to
# encode the query pattern using the same scheme used in carbon when they are written.
variants = [
TaggedSeries.encode(query.pattern, hash_only=True),
TaggedSeries.encode(query.pattern, hash_only=False),
]
else:
variants = extract_variants(query.pattern)
for variant in variants:
for fs_path in glob(self.tree.getFilesystemPath(variant)):
metric_path = self.tree.getNodePath(fs_path)
if CeresNode.isNodeDir(fs_path):
ceres_node = self.tree.getNode(metric_path)
if ceres_node.hasDataForInterval(query.startTime, query.endTime):
real_metric_path = get_real_metric_path(fs_path, metric_path)
reader = CeresReader(ceres_node, real_metric_path)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
yield LeafNode(metric_path, reader)
elif os.path.isdir(fs_path):
yield BranchNode(metric_path)
def test_taggedseries(self):
# test path with tags
parsed = TaggedSeries.parse('test.a;hello=tiger;blah=blah')
self.assertIsInstance(parsed, TaggedSeries)
self.assertEqual(parsed.metric, 'test.a')
self.assertEqual(parsed.tags, {'blah': 'blah', 'hello': 'tiger', 'name': 'test.a'})
# test formatting
self.assertEqual(parsed.path, 'test.a;blah=blah;hello=tiger')
# test encoding
self.assertEqual(TaggedSeries.encode(parsed.path), '_tagged.2b0.2af.test-a;blah=blah;hello=tiger')
# test path without tags
parsed = TaggedSeries.parse('test.a')
self.assertIsInstance(parsed, TaggedSeries)
self.assertEqual(parsed.metric, 'test.a')
self.assertEqual(parsed.tags, {'name': 'test.a'})
# test formatting
self.assertEqual(parsed.path, 'test.a')
# test encoding
self.assertEqual(TaggedSeries.encode(parsed.path), 'test.a')
# test parsing openmetrics
parsed = TaggedSeries.parse(r'test.a{hello="tiger",blah="bla\"h"}')
self.assertIsInstance(parsed, TaggedSeries)
self.assertEqual(parsed.metric, 'test.a')
self.assertEqual(parsed.tags, {'blah': 'bla"h', 'hello': 'tiger', 'name': 'test.a'})
def find_nodes(self, query):
# translate query pattern if it is tagged
tagged = not query.pattern.startswith(
'_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in ceres using encoded names, so to retrieve them we need to encode the
# query pattern using the same scheme used in carbon when they are written.
variants = [TaggedSeries.encode(query.pattern)]
else:
variants = extract_variants(query.pattern)
for variant in variants:
for fs_path in glob(self.tree.getFilesystemPath(variant)):
metric_path = self.tree.getNodePath(fs_path)
if CeresNode.isNodeDir(fs_path):
ceres_node = self.tree.getNode(metric_path)
if ceres_node.hasDataForInterval(query.startTime,
query.endTime):
real_metric_path = get_real_metric_path(
fs_path, metric_path)
reader = CeresReader(ceres_node, real_metric_path)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
yield LeafNode(metric_path, reader)
elif os.path.isdir(fs_path):
yield BranchNode(metric_path)
def get_series(self, path):
tags = {}
tags = self.r.hgetall('series:' + path + ':tags')
if not tags:
return None
return TaggedSeries(tags['name'], tags)
def get_series(self, path):
with connection.cursor() as cursor:
sql = 'SELECT s.id, t.tag, v.value'
sql += ' FROM tags_series AS s'
sql += ' JOIN tags_seriestag AS st ON st.series_id=s.id'
sql += ' JOIN tags_tag AS t ON t.id=st.tag_id'
sql += ' JOIN tags_tagvalue AS v ON v.id=st.value_id'
sql += ' WHERE s.path=%s'
params = [path]
cursor.execute(sql, params)
series_id = None
tags = {tag: value for (series_id, tag, value) in cursor.fetchall()}
if not tags:
return None
return TaggedSeries(tags['name'], tags, series_id=series_id)
def find_nodes(self, query):
clean_pattern = query.pattern.replace('\\', '')
# translate query pattern if it is tagged
tagged = not query.pattern.startswith('_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in whisper using encoded names, so to retrieve them we need to encode the
# query pattern using the same scheme used in carbon when they are written.
clean_pattern = TaggedSeries.encode(query.pattern)
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
for absolute_path in self._find_paths(root_dir, pattern_parts):
if basename(absolute_path).startswith('.'):
continue
if self.DATASOURCE_DELIMITER in basename(absolute_path):
(absolute_path, datasource_pattern) = absolute_path.rsplit(
self.DATASOURCE_DELIMITER, 1)
else:
datasource_pattern = None
relative_path = absolute_path[len(root_dir):].lstrip('/')
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(query.pattern):
metric_path_parts[field_index] = pattern_parts[field_index].replace(
'\\', '')
metric_path = '.'.join(metric_path_parts)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif isfile(absolute_path):
if absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(
absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries(
[datasource_name], datasource_pattern):
reader = RRDReader(
absolute_path, datasource_name)
yield LeafNode(metric_path + "." + datasource_name, reader)
def parse(path): return TaggedSeries.parse(path)
def parse(path):
return TaggedSeries.parse(path)
def find_nodes(self, query):
clean_pattern = query.pattern.replace('\\', '')
# translate query pattern if it is tagged
tagged = not query.pattern.startswith(
'_tagged.') and ';' in query.pattern
if tagged:
# tagged series are stored in whisper using encoded names, so to retrieve them we need to
# encode the query pattern using the same scheme used in carbon when they are written.
encoded_paths = [
TaggedSeries.encode(query.pattern, sep=os.sep, hash_only=True),
TaggedSeries.encode(query.pattern, sep=os.sep,
hash_only=False),
]
pattern_parts = clean_pattern.split('.')
for root_dir in self.directories:
if tagged:
relative_paths = []
for pattern in encoded_paths:
entries = [
pattern + '.wsp',
pattern + '.wsp.gz',
pattern + '.rrd',
]
for entry in entries:
if isfile(join(root_dir, entry)):
relative_paths.append(entry)
else:
relative_paths = self._find_paths(root_dir, pattern_parts)
for relative_path in relative_paths:
if basename(relative_path).startswith('.'):
continue
if self.DATASOURCE_DELIMITER in basename(relative_path):
(relative_path, datasource_pattern) = relative_path.rsplit(
self.DATASOURCE_DELIMITER, 1)
else:
datasource_pattern = None
absolute_path = join(root_dir, relative_path)
metric_path = fs_to_metric(relative_path)
real_metric_path = get_real_metric_path(
absolute_path, metric_path)
# if we're finding by tag, return the proper metric path
if tagged:
metric_path = query.pattern
else:
metric_path_parts = metric_path.split('.')
for field_index in find_escaped_pattern_fields(
query.pattern):
metric_path_parts[field_index] = pattern_parts[
field_index].replace('\\', '')
metric_path = '.'.join(metric_path_parts)
# Now we construct and yield an appropriate Node object
if isdir(absolute_path):
yield BranchNode(metric_path)
elif absolute_path.endswith(
'.wsp') and WhisperReader.supported:
reader = WhisperReader(absolute_path, real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith(
'.wsp.gz') and GzippedWhisperReader.supported:
reader = GzippedWhisperReader(absolute_path,
real_metric_path)
yield LeafNode(metric_path, reader)
elif absolute_path.endswith('.rrd') and RRDReader.supported:
if datasource_pattern is None:
yield BranchNode(metric_path)
else:
for datasource_name in RRDReader.get_datasources(
absolute_path):
if match_entries([datasource_name],
datasource_pattern):
reader = RRDReader(absolute_path,
datasource_name)
yield LeafNode(
metric_path + "." + datasource_name,
reader)
