def test_query_only_statement(self):
dummy = self.dummyQuery(
statement='list',
ast=('match_and', ('match_tag_exists', 'target_type'),
('match_tag_exists', 'unit')),
patterns=['target_type=', 'unit='],
target_modifiers=[Query.derive_counters],
)
query = Query("list")
self.assertQueryMatches(query, dummy)
query = Query("list ")
self.assertQueryMatches(query, dummy)
def test_sum_by_buckets(self):
query = Query(
"stack from -20hours to -10hours avg over 10M sum by foo:bucket1|bucket2,bar min 100 max 200"
)
self.assertQueryMatches(
query,
self.dummyQuery(
**{
'ast': ('match_and', ('match_tag_exists', 'target_type'),
('match_tag_exists', 'unit')),
'patterns': ['target_type=', 'unit='],
'statement':
'stack',
'avg_over': (10, 'M'),
'from':
'-20hours',
'to':
'-10hours',
'min':
'100',
'max':
'200',
'sum_by': {
'foo': ['bucket1', 'bucket2', ''],
'bar': ['']
},
'target_modifiers': [Query.derive_counters],
}))
def test_group_by_advanced(self):
query = Query(
"dfvimeodfs disk srv node used group by mountpoint=:dfs1,server")
# note: ideally, the order would be <default group by strong> + user defined group by's
# but that was a little hard to implement
self.assertQueryMatches(
query,
self.dummyQuery(
**{
'ast': ('match_and', ('match_tag_exists', 'mountpoint'),
('match_tag_exists',
'target_type'), ('match_tag_exists', 'unit'),
('match_id_regex',
'dfvimeodfs'), ('match_id_regex', 'disk'),
('match_id_regex',
'srv'), ('match_id_regex',
'node'), ('match_id_regex', 'used')),
'patterns': [
'mountpoint=', 'target_type=', 'unit=', 'dfvimeodfs',
'disk', 'srv', 'node', 'used'
],
'group_by': {
'target_type': [''],
'unit': [''],
'mountpoint': ['dfs1', ''],
'server': ['']
},
'target_modifiers': [Query.derive_counters],
}))
def test_empty(self):
query = Query("")
self.assertQueryMatches(
query,
self.dummyQuery(ast=('match_and', ('match_tag_exists',
'target_type'),
('match_tag_exists', 'unit')),
target_modifiers=[Query.derive_counters],
patterns=['target_type=', 'unit=']))
def test_two_simple_terms(self):
query = Query("foo bar")
self.assertQueryMatches(
query,
self.dummyQuery(ast=('match_and', ('match_tag_exists',
'target_type'),
('match_tag_exists',
'unit'), ('match_id_regex',
'foo'), ('match_id_regex', 'bar')),
target_modifiers=[Query.derive_counters],
patterns=['target_type=', 'unit=', 'foo', 'bar']))
def test_typo_before_sum(self):
query = Query(
"octo -20hours unit=b/s memory group by foo avg by barsum by baz")
dummy = self.dummyQuery(avg_by={'barsum': ['']},
group_by={
'target_type': [''],
'unit': [''],
'foo': ['']
},
patterns=[
'target_type=', 'unit=', 'octo',
'-20hours', 'unit=b/s', 'memory', 'by',
'baz'
])
del dummy['target_modifiers']
self.assertDictContainsSubset(dummy, query)
ast_first_part = (
'match_and',
('match_tag_exists', 'target_type'),
('match_tag_exists', 'unit'),
('match_id_regex', 'octo'),
('match_id_regex', '-20hours'),
)
ast_last_part = (('match_id_regex', 'memory'),
('match_id_regex', 'by'), ('match_id_regex', 'baz'))
ast = query['ast']
self.assertTupleEqual(ast[:len(ast_first_part)], ast_first_part)
self.assertTupleEqual(ast[len(ast_first_part) + 1:], ast_last_part)
fat_hairy_or_filter = ast[len(ast_first_part)]
self.assertEqual(fat_hairy_or_filter[0], 'match_or')
unit_clauses = fat_hairy_or_filter[1:]
for clause in unit_clauses:
self.assertEqual(clause[:2], ('match_tag_equality', 'unit'))
all_the_units = [clause[2] for clause in unit_clauses]
for unit in ('b/s', 'MiB/s', 'PiB', 'kB/w', 'b'):
self.assertIn(unit, all_the_units)
self.assertTrue(any('apply_requested_unit' in str(f)
for f in query['target_modifiers']),
msg='apply_requested_unit callback not in %r' %
query['target_modifiers'])
def test_build_buckets_one_no_buckets():
assert Query.build_buckets("foo") == {'foo': ['']}
def test_build_buckets_two_with_buckets_group_by_style():
# for 'group by', there can be '=' in there.
assert Query.build_buckets('target_type=,region:us-east|us-west|') == {
'target_type=': [''],
'region': ['us-east', 'us-west', '']
}
def test_build_buckets_two_with_buckets_group_by_style():
# for 'group by', there can be '=' in there.
assert Query.build_buckets('target_type=,region:us-east|us-west|') == {
'target_type=': [''],
'region': ['us-east', 'us-west', '']
}
def test_build_buckets_two_with_buckets():
assert Query.build_buckets("n3:bucketmatch1|bucketmatch2,othertag") == {
'n3': ['bucketmatch1', 'bucketmatch2', ''],
'othertag': ['']
}
def test_aggregation():
preferences = DummyPrefs()
# note: uneven aggregation: we only want 1 resulting metric,
query = Query("")
query['avg_by'] = {'server': ['']}
query['sum_by'] = {'type': ['']}
targets = {
'web1.db': {
'id': 'web1.db',
'tags': {
'server': 'web1',
'type': 'db',
'n3': 'foo'
}
},
'web1.php': {
'id': 'web1.php',
'tags': {
'server': 'web1',
'type': 'php',
'n3': 'foo'
}
},
'web2.db': {
'id': 'web2.db',
'tags': {
'server': 'web2',
'type': 'db',
'n3': 'foo'
}
},
'web2.php': {
'id': 'web2.php',
'tags': {
'server': 'web2',
'type': 'php',
'n3': 'foo'
}
},
'web2.memcache': {
'id': 'web2.memcache',
'tags': {
'server': 'web2',
'type': 'memcache',
'n3': 'foo'
}
}
}
from pprint import pprint
for (k, v) in targets.items():
v = Target(v)
v.get_graph_info(group_by={})
targets[k] = v
graphs, _query = g.build_from_targets(targets, query, preferences)
# TODO: there should be only 1 graph, containing all 5 items
print "Graphs:"
for (k, v) in graphs.items():
print "graph key"
pprint(k)
print "val:"
pprint(v)
assert {} == graphs
def test_nontrivial_implicit_aggregation():
preferences = DummyPrefs()
# we ultimately want 1 graph with 1 line for each server,
# irrespective of the values of the other tags (n1 and n2)
# and even whether or not the metrics have those tags at all.
query = Query("")
query['group_by'] = {}
query['sum_by'] = {'n1': [''], 'n2': ['']}
targets = {
# web1 : one with and without n2
'web1.a.a': {
'id': 'web1.a.a',
'tags': {
'server': 'web1',
'n1': 'a',
'n2': 'a'
}
},
'web1.a': {
'id': 'web1.a',
'tags': {
'server': 'web1',
'n1': 'a',
}
},
# web 2: 2 different values of n2
'web2.a.a': {
'id': 'web2.a.a',
'tags': {
'server': 'web2',
'n1': 'a',
'n2': 'a'
}
},
'web2.a.b': {
'id': 'web2.a.b',
'tags': {
'server': 'web2',
'n1': 'a',
'n2': 'b'
}
},
# web3: with and without n2, diff value for n1
'web3.a.a': {
'id': 'web3.a.a',
'tags': {
'server': 'web3',
'n1': 'a',
'n2': 'a'
}
},
'web3.b': {
'id': 'web3.b',
'tags': {
'server': 'web3',
'n1': 'b'
}
}
}
from pprint import pprint
for (k, v) in targets.items():
v = Target(v)
v.get_graph_info(group_by={})
targets[k] = v
graphs, _query = g.build_from_targets(targets, query, preferences)
# TODO: there should be only 1 graph, containing 3 lines, with each 2 targets per server
# i.e. something like this:
expected = {
'targets': {
'web1.a.a__web1.a': {
'id': ['web1.a.a', 'web1.a']
},
'web2.a.a__web2.a.b': {
'id': ['web2.a.a', 'web2.a.b']
},
'web3.a.a__web3.b': {
'id': ['web3.a.a', 'web3.b']
}
}
}
print "Graphs:"
for (k, v) in graphs.items():
print "graph key"
pprint(k)
print "val:"
pprint(v)
assert expected == graphs
def test_build_buckets_two_no_buckets():
assert Query.build_buckets("foo,bar") == {'foo': [''], 'bar': ['']}
def test_build_buckets_one_no_buckets():
assert Query.build_buckets("foo") == {'foo': ['']}
def test_build_buckets_two_no_buckets():
assert Query.build_buckets("foo,bar") == {'foo': [''], 'bar': ['']}
def test_equivalence():
preferences = DummyPrefs()
query = Query("")
query['sum_by'] = {'core': ['']}
targets = {
'servers.host.cpu.cpu0.irq': {
'id': 'servers.host.cpu.cpu0.irq',
'tags': {
'core': 'cpu0',
'plugin': 'cpu',
'server': 'host',
'target_type': 'gauge_pct',
'type': 'irq',
'unit': 'cpu_state'
}
},
'servers.host.cpu.cpu0.softirq': {
'id': 'servers.host.cpu.cpu0.softirq',
'tags': {
'core': 'cpu0',
'plugin': 'cpu',
'server': 'host',
'target_type': 'gauge_pct',
'type': 'softirq',
'unit': 'cpu_state'
}
},
'servers.host.cpu.cpu2.irq': {
'id': 'servers.host.cpu.cpu2.irq',
'tags': {
'core': 'cpu2',
'plugin': 'cpu',
'server': 'host',
'target_type': 'gauge_pct',
'type': 'irq',
'unit': 'cpu_state'
}
},
'servers.host.cpu.cpu2.softirq': {
'id': 'servers.host.cpu.cpu2.softirq',
'tags': {
'core': 'cpu2',
'plugin': 'cpu',
'server': 'host',
'target_type': 'gauge_pct',
'type': 'softirq',
'unit': 'cpu_state'
}
},
'servers.host.cpu.total.irq': {
'id': 'servers.host.cpu.total.irq',
'tags': {
'core': '_sum_',
'plugin': 'cpu',
'server': 'host',
'target_type': 'gauge_pct',
'type': 'irq',
'unit': 'cpu_state'
}
},
'servers.host.cpu.total.softirq': {
'id': 'servers.host.cpu.total.softirq',
'tags': {
'core': '_sum_',
'plugin': 'cpu',
'server': 'host',
'target_type': 'gauge_pct',
'type': 'softirq',
'unit': 'cpu_state'
}
}
}
graphs, _query = g.build_from_targets(targets, query, preferences)
assert len(graphs) == 1
_, graph = graphs.popitem()
assert len(graph['targets']) == 2
ids = [t['id'] for t in graph['targets']]
assert ids == [
'servers.host.cpu.total.irq', 'servers.host.cpu.total.softirq'
]
# if there's a filter, equivalence doesn't hold and we should get 2 targets,
# each the sum of two non-sums
# and the _sum_ metrics should be removed
query = Query("core:(_sum_|cpu0|cpu2) sum by core")
#query['sum_by'] = {'core': ['']}
#query['patterns'].append('core:(_sum_|cpu0|cpu2)')
graphs, _query = g.build_from_targets(targets, query, preferences)
assert len(graphs) == 1
_, graph = graphs.popitem()
assert len(graph['targets']) == 2
ids = [t['id'] for t in graph['targets']]
assert ids == [[
'servers.host.cpu.cpu0.softirq', 'servers.host.cpu.cpu2.softirq'
], ['servers.host.cpu.cpu0.irq', 'servers.host.cpu.cpu2.irq']]
def test_build_buckets_two_with_buckets():
assert Query.build_buckets("n3:bucketmatch1|bucketmatch2,othertag") == {
'n3': ['bucketmatch1', 'bucketmatch2', ''],
'othertag': ['']
}