def test_standard_finder_gzipped_whisper(self):
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
self.create_whisper('foo.wsp', True)
self.create_whisper(join('foo', 'bar', 'baz.wsp'), True)
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
finally:
os.listdir = self._original_listdir
self.wipe_whisper()
def test_finder_nodes(self):
'''
Test that the finder can find nodes.
'''
# Base query
nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1', 'branch2', 'leaf'],
)
# One level down
nodes = list(self.finder.find_nodes(query=FindQuery('*.leaf', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1.leaf', 'branch2.leaf']
)
nodes = list(self.finder.find_nodes(query=FindQuery('branch1.leaf', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1.leaf'],
)
def query_test(query_pattern, fg_data, search_results, bg_data=[]):
# query_pattern is the pattern to search for
# fg_data is the data returned by the "mainthread" call to find metrics
# search_results are the expected results
# bg_data - non submetric calls do 2 calls to find_metrics, (one in a background thread,)
# bg_data simulates what gets returnd by the background thread
def json_callback(request, context):
print("json thread callback" + threading.current_thread().name)
full_query = "include_enum_values=true&" + urllib.urlencode({
'query':
query_pattern
}).lower()
if not self.finder.enable_submetrics:
if request.query == full_query:
return fg_data
else:
return bg_data
else:
return fg_data
qlen = len(query_pattern.split("."))
with requests_mock.mock() as m:
query = FindQuery(query_pattern, 1, 2)
m.get(endpoint, json=json_callback, status_code=200)
metrics = self.finder.find_nodes(query)
self.assertSetEqual(set(map(get_path, list(metrics))),
set(map(get_start(qlen), search_results)))
def query_test(query_pattern, jdata, qlen, search_results):
with requests_mock.mock() as m:
query = FindQuery(query_pattern, 1, 2)
m.get(endpoint, json=jdata, status_code=200)
metrics = self.finder.find_nodes(query)
self.assertSequenceEqual(map(get_path, list(metrics)),
map(get_start(qlen), search_results))
def test_standard_finder(self):
test_dir = join(settings.WHISPER_DIR, 'test_standard_finder')
def create_whisper(path):
path = join(test_dir, path)
if not isdir(dirname(path)):
os.makedirs(dirname(path))
whisper.create(path, [(1, 60)])
def wipe_whisper():
try:
shutil.rmtree(test_dir)
except OSError:
pass
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
create_whisper('foo.wsp')
create_whisper(join('foo', 'bar', 'baz.wsp'))
create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(
FindQuery('test_standard_finder.foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(
FindQuery('test_standard_finder.foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(
FindQuery('test_standard_finder.*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
finally:
os.listdir = self._original_listdir
wipe_whisper()
def test_finder_missing_branch(self):
'''
Test that the finder can deal with a missing branch.
'''
finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 3)
with self.assertRaises(ValueError):
nodes = list(finder.find_nodes(query=FindQuery('*', None, None)))
def test_caching(self):
'''
Test caching behaviour.
'''
with HTTMock(mocked_urls):
nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1', 'branch2', 'leaf'],
)
with HTTMock(bad_urls):
nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1', 'branch2', 'leaf'],
)
def test_finder_character_classes(self):
'''
Test that the finder can deal with character classes.
'''
nodes = list(self.finder.find_nodes(query=FindQuery('branch[1]', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1'],
)
def test_finder_braces(self):
'''
Test that the finder can deal with brace expressions.
'''
nodes = list(self.finder.find_nodes(query=FindQuery('{branch1,leaf}', None, None)))
self.assertEqual(
[node.path for node in nodes],
['branch1', 'leaf'],
)
def test_standard_finder_gzipped_whisper(self, scandir_mock):
try:
self.create_whisper('foo.wsp', True)
self.create_whisper(join('foo', 'bar', 'baz.wsp'), True)
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
finally:
scandir_mock.call_count = 0
self.wipe_whisper()
def test_finder_nested_dotted_nodes(self):
'''
Test that the finder can deal with nested nodes with dots
(Since OpenTSDB allows that).
'''
finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 2)
nodes = list(finder.find_nodes(query=FindQuery('*.*', None, None)))
self.assertEqual(
[node.name for node in nodes],
['leaf.with.dots'],
)
def query_test(query_pattern, fg_data, search_results):
# query_pattern is the pattern to search for
# fg_data -> data returned by the "mainthread" call to find metrics
# search_results are the expected results
def json_callback(request, context):
print("json thread callback" + threading.current_thread().name)
return fg_data
qlen = len(query_pattern.split("."))
with requests_mock.mock() as m:
query = FindQuery(query_pattern, 1, 2)
m.get(endpoint, json=json_callback, status_code=200)
metrics = self.finder.find_nodes(query)
self.assertSetEqual(set(map(get_path, list(metrics))),
set(map(get_start(qlen), search_results)))
def test_standard_finder_tagged_whisper_carbonlink(self):
try:
self.create_whisper(join(
'_tagged',
'b34',
'2de',
# foo;bar=baz2
'b342defa10cb579981c63ef78be5ac248f681f4bd2c35bc0209d3a7b9eb99346.wsp'
))
finder = get_finders('graphite.finders.standard.StandardFinder')[0]
nodes = list(finder.find_nodes(FindQuery('foo;bar=baz2', None, None)))
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].reader.real_metric_path, 'foo;bar=baz2')
finally:
self.wipe_whisper()
def test_ceres_finder(self):
test_dir = join(settings.CERES_DIR)
def create_ceres(metric):
if not isdir(test_dir):
os.makedirs(test_dir)
tree = ceres.CeresTree(test_dir)
options = {}
tree.createNode(metric, **options)
tree.store(metric, [(1, 60)])
def wipe_ceres():
try:
shutil.rmtree(test_dir)
except OSError:
pass
self.addCleanup(wipe_ceres)
create_ceres('foo')
create_ceres('foo.bar.baz')
create_ceres('bar.baz.foo')
finder = get_finders('graphite.finders.ceres.CeresFinder')[0]
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 1)
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
# No data in the expected time period
nodes = finder.find_nodes(FindQuery('foo.bar.baz', 10000, 10060))
self.assertEqual(len(list(nodes)), 0)
nodes = finder.find_nodes(FindQuery('foo.bar', None, None))
self.assertEqual(len(list(nodes)), 1)
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
# Search for something that isn't valid Ceres content
fh = open(join(test_dir, 'foo', 'blah'), 'wb')
fh.close()
nodes = finder.find_nodes(FindQuery('foo.blah', None, None))
self.assertEqual(len(list(nodes)), 0)
# get index
result = finder.get_index({})
self.assertEqual(result, ['bar.baz.foo', 'foo', 'foo.bar.baz'])
def query_test_with_submetrics(query_pattern,
fg_data,
search_results,
bg_data=[]):
# query_pattern is the pattern to search for
# fg_data -> data returned by the "mainthread" call to find metrics
# search_results are the expected results
# bg_data - non submetric calls do 2 calls to find_metrics, (one
# in a background thread,)
# bg_data simulates what gets returnd by the background thread
def json_callback(request, context):
print("json thread callback" + threading.current_thread().name)
return fg_data
qlen = len(query_pattern.split("."))
with requests_mock.mock() as m:
query = FindQuery(query_pattern, 1, 2)
m.get(endpoint_old, json=json_callback, status_code=200)
metrics = self.finder.find_nodes(query)
self.assertSetEqual(set(map(get_path, list(metrics))),
set(map(get_start(qlen), search_results)))
enum_vals = ['v1', 'v2']
query_test("*", [{'a': False}, {'b': False}], ['a', 'b'])
query_test("a.*", [{'a.b': False}], ['a.b'])
query_test("a.b.*", [{'a.b.c': True}], [self.metric1])
query_test("a.b.c", [{'a.b.c': True}], [self.metric1])
query_test("a.b.c.*", [{
'a.b.c.d': False
}, {
'a.b.c.v1': True
}, {
'a.b.c.v2': True
}], [self.metric1 + '.' + v
for v in enum_vals] + [self.metric1 + '.d'])
query_test("a.b.*.v*", [{
'a.b.c.v': False
}, {
'a.b.c.v1': True
}, {
'a.b.c.v2': True
}], [self.metric1 + '.' + v
for v in enum_vals] + [self.metric1 + '.v.e'])
query_test("a.b.*.v*.*", [{
'a.b.c.v.e': False
}], [self.metric1 + '.v.e.f'])
# now again, with submetrics
self.finder.enable_submetrics = True
query_test_with_submetrics("*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': self.metric2,
u'unit': u'percent'
}], [self.metric1, self.metric2])
query_test_with_submetrics("a.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test_with_submetrics("a.*", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1])
query_test_with_submetrics("a.b.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': 'a.bb.c',
u'unit': u'percent'
}], [self.metric1])
query_test_with_submetrics("a.b.c", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test_with_submetrics("a.b.c.*", [
{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': (self.metric1 + 'd'),
u'unit': u'percent'
}
], [self.metric1 + '.' + k for k in self.finder.submetric_aliases])
query_test_with_submetrics("a.b.c.*", [
{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}, {
u'metric': (self.metric1 + 'd'),
u'unit': u'percent'
}
], [self.metric1 + '.' + k for k in self.finder.submetric_aliases])
query_test_with_submetrics("a.b.c._avg", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1 + '.' + self.alias_key])
query_test_with_submetrics("a.b.c._avg", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + self.alias_key])
query_test_with_submetrics("a.b.c.v1._enum",
[{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.v1'])
query_test_with_submetrics(
"a.b.c.*._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + v for v in enum_vals])
query_test_with_submetrics(
"a.b.*.*._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + v for v in enum_vals])
query_test_with_submetrics(
"a.b.c.v*._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + v for v in enum_vals])
def test_standard_finder(self, scandir_mock):
try:
self.create_whisper('foo.wsp')
self.create_whisper(join('foo', 'bar', 'baz.wsp'))
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
self.create_whisper(
join('_tagged', '9c6', '79b', 'foo;bar=baz.wsp'))
self.create_whisper(
join(
'_tagged',
'b34',
'2de',
# foo;bar=baz2
'b342defa10cb579981c63ef78be5ac248f681f4bd2c35bc0209d3a7b9eb99346.wsp'
))
finder = get_finders('graphite.finders.standard.StandardFinder')[0]
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 4)
scandir_mock.call_count = 0
nodes = finder.find_nodes(
FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(
FindQuery('foo.{ba{r,z},baz}.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(
FindQuery('{foo,garbage}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(
FindQuery('{fo,ba}{o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(
FindQuery('{fo,ba}{o,o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(
FindQuery('{fo,ba}{o,z}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo;bar=baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo;bar=baz2', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
results = finder.fetch(['foo'], 0, 1)
self.assertEqual(results, [])
finally:
scandir_mock.call_count = 0
self.wipe_whisper()
def run_find(self, finder):
nodes = list(finder.find_nodes(FindQuery('rackspace.*', 0, 100)))
self.assertTrue(len(nodes) > 0)
def test_standard_finder(self):
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
self.create_whisper('foo.wsp')
self.create_whisper(join('foo', 'bar', 'baz.wsp'))
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 2)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
finally:
os.listdir = self._original_listdir
self.wipe_whisper()
def test_finder(self):
finder = TenantBluefloodFinder(config)
nodes = list(finder.find_nodes(FindQuery('rackspace.*', 0, 100)))
self.assertTrue(len(nodes) > 0)
def test_standard_finder(self,scandir_mock):
try:
self.create_whisper('foo.wsp')
self.create_whisper(join('foo', 'bar', 'baz.wsp'))
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 5)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 5)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 2)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
finally:
scandir_mock.call_count = 0
self.wipe_whisper()
def test_find_nodes(self):
endpoint = self.finder.find_metrics_endpoint(
self.finder.bf_query_endpoint, self.finder.tenant)
# one time through without submetrics
self.finder.enable_submetrics = False
with requests_mock.mock() as m:
#test 401 errors
query = FindQuery("*", 1, 2)
m.get(endpoint, json={}, status_code=401)
metrics = self.finder.find_nodes(query)
self.assertTrue(list(metrics) == [])
with requests_mock.mock() as m:
query = FindQuery("*", 1, 2)
m.get(endpoint, json=exc_callback, status_code=401)
with self.assertRaises(ValueError):
list(self.finder.find_nodes(query))
def get_start(x):
return lambda y: '.'.join(y.split('.')[:x])
get_path = lambda x: x.path
def query_test(query_pattern, fg_data, search_results, bg_data=[]):
# query_pattern is the pattern to search for
# fg_data is the data returned by the "mainthread" call to find metrics
# search_results are the expected results
# bg_data - non submetric calls do 2 calls to find_metrics, (one in a background thread,)
# bg_data simulates what gets returnd by the background thread
def json_callback(request, context):
print("json thread callback" + threading.current_thread().name)
full_query = "include_enum_values=true&" + urllib.urlencode({
'query':
query_pattern
}).lower()
if not self.finder.enable_submetrics:
if request.query == full_query:
return fg_data
else:
return bg_data
else:
return fg_data
qlen = len(query_pattern.split("."))
with requests_mock.mock() as m:
query = FindQuery(query_pattern, 1, 2)
m.get(endpoint, json=json_callback, status_code=200)
metrics = self.finder.find_nodes(query)
self.assertSetEqual(set(map(get_path, list(metrics))),
set(map(get_start(qlen), search_results)))
enum_vals = ['v1', 'v2']
query_test("*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': self.metric2,
u'unit': u'percent'
}], [self.metric1, self.metric2])
query_test("a.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test("a.*", [{
u'metric': self.metric1 + 'd',
u'unit': u'percent'
}], [self.metric1], [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}])
query_test("a.b.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test("a.b.c", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test("a.b.c.*", [{
u'metric': self.metric1 + '.d.e',
u'unit': u'percent'
}], [self.metric1 + '.' + v
for v in enum_vals] + [self.metric1 + '.d.e'],
[{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}])
query_test("a.b.*.v*", [{
u'metric': self.metric1 + '.v.e',
u'unit': u'percent'
}], [self.metric1 + '.' + v
for v in enum_vals] + [self.metric1 + '.v.e'],
[{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}])
query_test("a.b.*.v*.*", [{
u'metric': self.metric1 + '.v.e.f',
u'unit': u'percent'
}], [self.metric1 + '.v.e.f'], [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}])
# now again, with submetrics
self.finder.enable_submetrics = True
query_test("*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': self.metric2,
u'unit': u'percent'
}], [self.metric1, self.metric2])
query_test("a.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test("a.*", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1])
query_test("a.b.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': 'a.bb.c',
u'unit': u'percent'
}], [self.metric1])
query_test("a.b.c", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1])
query_test("a.b.c.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': (self.metric1 + 'd'),
u'unit': u'percent'
}], [self.metric1 + '.' + k for k in self.finder.submetric_aliases])
query_test("a.b.c.*", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}, {
u'metric': (self.metric1 + 'd'),
u'unit': u'percent'
}], [self.metric1 + '.' + k for k in self.finder.submetric_aliases])
query_test("a.b.c._avg", [{
u'metric': self.metric1,
u'unit': u'percent'
}], [self.metric1 + '.' + self.alias_key])
query_test("a.b.c._avg", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + self.alias_key])
query_test("a.b.c.v1._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.v1'])
query_test("a.b.c.*._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + v for v in enum_vals])
query_test("a.b.*.*._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + v for v in enum_vals])
query_test("a.b.c.v*._enum", [{
u'metric': self.metric1,
u'unit': u'percent',
u'enum_values': enum_vals
}], [self.metric1 + '.' + v for v in enum_vals])
def test_ceres_finder(self):
test_dir = join(settings.CERES_DIR)
def create_ceres(metric):
if not isdir(test_dir):
os.makedirs(test_dir)
tree = ceres.CeresTree(test_dir)
options = {}
tree.createNode(metric, **options)
tree.store(metric, [(1, 60)])
def wipe_ceres():
try:
shutil.rmtree(test_dir)
except OSError:
pass
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
create_ceres('foo')
create_ceres('foo.bar.baz')
create_ceres('bar.baz.foo')
finder = get_finder('graphite.finders.ceres.CeresFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
# No data in the expected time period
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', 10000, 10060))
self.assertEqual(len(list(nodes)), 0)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 8)
# Search for something that isn't valid Ceres content
fh = open(join(test_dir, 'foo', 'blah'), 'wb')
fh.close()
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.blah', None, None))
self.assertEqual(len(list(nodes)), 0)
self.assertEqual(self._listdir_counter, 0)
finally:
os.listdir = self._original_listdir
wipe_ceres()
def test_find_nodes(self):
endpoint = self.finder.find_metrics_endpoint(
self.finder.bf_query_endpoint, self.finder.tenant)
# one time through without submetrics
self.finder.enable_submetrics = False
with requests_mock.mock() as m:
#test 401 errors
query = FindQuery("*", 1, 2)
m.get(endpoint, json={}, status_code=401)
metrics = self.finder.find_nodes(query)
self.assertTrue(list(metrics) == [])
with requests_mock.mock() as m:
query = FindQuery("*", 1, 2)
m.get(endpoint, json=exc_callback, status_code=401)
with self.assertRaises(ValueError):
list(self.finder.find_nodes(query))
def get_start(x):
return lambda y: '.'.join(y.split('.')[0:x])
get_path = lambda x: x.path
def query_test(query_pattern, jdata, qlen, search_results):
with requests_mock.mock() as m:
query = FindQuery(query_pattern, 1, 2)
m.get(endpoint, json=jdata, status_code=200)
metrics = self.finder.find_nodes(query)
self.assertSequenceEqual(map(get_path, list(metrics)),
map(get_start(qlen), search_results))
query_test("*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': self.metric2,
u'unit': u'percent'
}], 1, [self.metric1, self.metric2])
query_test("a.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], 2, [self.metric1])
query_test("a.b.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], 3, [self.metric1])
query_test("a.b.c", [{
u'metric': self.metric1,
u'unit': u'percent'
}], 3, [self.metric1])
# now again, with submetrics
self.finder.enable_submetrics = True
query_test("*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': self.metric2,
u'unit': u'percent'
}], 1, [self.metric1, self.metric2])
query_test("a.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}], 2, [self.metric1])
query_test("a.b.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': 'a.bb.c',
u'unit': u'percent'
}], 3, [self.metric1])
query_test("a.b.c", [{
u'metric': self.metric1,
u'unit': u'percent'
}], 3, [self.metric1])
query_test("a.b.c.*", [{
u'metric': self.metric1,
u'unit': u'percent'
}, {
u'metric': (self.metric1 + 'd'),
u'unit': u'percent'
}], 4, [self.metric1 + '.' + self.alias_key])
query_test("a.b.c._avg", [{
u'metric': self.metric1,
u'unit': u'percent'
}], 4, [self.metric1 + '.' + self.alias_key])
