def test_standard_finder_gzipped_whisper(self): def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock self.create_whisper('foo.wsp', True) self.create_whisper(join('foo', 'bar', 'baz.wsp'), True) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) finally: os.listdir = self._original_listdir self.wipe_whisper()
def test_finder_nodes(self): ''' Test that the finder can find nodes. ''' # Base query nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1', 'branch2', 'leaf'], ) # One level down nodes = list(self.finder.find_nodes(query=FindQuery('*.leaf', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1.leaf', 'branch2.leaf'] ) nodes = list(self.finder.find_nodes(query=FindQuery('branch1.leaf', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1.leaf'], )
def query_test(query_pattern, fg_data, search_results, bg_data=[]): # query_pattern is the pattern to search for # fg_data is the data returned by the "mainthread" call to find metrics # search_results are the expected results # bg_data - non submetric calls do 2 calls to find_metrics, (one in a background thread,) # bg_data simulates what gets returnd by the background thread def json_callback(request, context): print("json thread callback" + threading.current_thread().name) full_query = "include_enum_values=true&" + urllib.urlencode({ 'query': query_pattern }).lower() if not self.finder.enable_submetrics: if request.query == full_query: return fg_data else: return bg_data else: return fg_data qlen = len(query_pattern.split(".")) with requests_mock.mock() as m: query = FindQuery(query_pattern, 1, 2) m.get(endpoint, json=json_callback, status_code=200) metrics = self.finder.find_nodes(query) self.assertSetEqual(set(map(get_path, list(metrics))), set(map(get_start(qlen), search_results)))
def query_test(query_pattern, jdata, qlen, search_results): with requests_mock.mock() as m: query = FindQuery(query_pattern, 1, 2) m.get(endpoint, json=jdata, status_code=200) metrics = self.finder.find_nodes(query) self.assertSequenceEqual(map(get_path, list(metrics)), map(get_start(qlen), search_results))
def test_standard_finder(self): test_dir = join(settings.WHISPER_DIR, 'test_standard_finder') def create_whisper(path): path = join(test_dir, path) if not isdir(dirname(path)): os.makedirs(dirname(path)) whisper.create(path, [(1, 60)]) def wipe_whisper(): try: shutil.rmtree(test_dir) except OSError: pass def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock create_whisper('foo.wsp') create_whisper(join('foo', 'bar', 'baz.wsp')) create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes( FindQuery('test_standard_finder.foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes( FindQuery('test_standard_finder.foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes( FindQuery('test_standard_finder.*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) finally: os.listdir = self._original_listdir wipe_whisper()
def test_finder_missing_branch(self): ''' Test that the finder can deal with a missing branch. ''' finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 3) with self.assertRaises(ValueError): nodes = list(finder.find_nodes(query=FindQuery('*', None, None)))
def test_caching(self): ''' Test caching behaviour. ''' with HTTMock(mocked_urls): nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1', 'branch2', 'leaf'], ) with HTTMock(bad_urls): nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1', 'branch2', 'leaf'], )
def test_finder_character_classes(self): ''' Test that the finder can deal with character classes. ''' nodes = list(self.finder.find_nodes(query=FindQuery('branch[1]', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1'], )
def test_finder_braces(self): ''' Test that the finder can deal with brace expressions. ''' nodes = list(self.finder.find_nodes(query=FindQuery('{branch1,leaf}', None, None))) self.assertEqual( [node.path for node in nodes], ['branch1', 'leaf'], )
def test_standard_finder_gzipped_whisper(self, scandir_mock): try: self.create_whisper('foo.wsp', True) self.create_whisper(join('foo', 'bar', 'baz.wsp'), True) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) finally: scandir_mock.call_count = 0 self.wipe_whisper()
def test_finder_nested_dotted_nodes(self): ''' Test that the finder can deal with nested nodes with dots (Since OpenTSDB allows that). ''' finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 2) nodes = list(finder.find_nodes(query=FindQuery('*.*', None, None))) self.assertEqual( [node.name for node in nodes], ['leaf.with.dots'], )
def query_test(query_pattern, fg_data, search_results): # query_pattern is the pattern to search for # fg_data -> data returned by the "mainthread" call to find metrics # search_results are the expected results def json_callback(request, context): print("json thread callback" + threading.current_thread().name) return fg_data qlen = len(query_pattern.split(".")) with requests_mock.mock() as m: query = FindQuery(query_pattern, 1, 2) m.get(endpoint, json=json_callback, status_code=200) metrics = self.finder.find_nodes(query) self.assertSetEqual(set(map(get_path, list(metrics))), set(map(get_start(qlen), search_results)))
def test_standard_finder_tagged_whisper_carbonlink(self): try: self.create_whisper(join( '_tagged', 'b34', '2de', # foo;bar=baz2 'b342defa10cb579981c63ef78be5ac248f681f4bd2c35bc0209d3a7b9eb99346.wsp' )) finder = get_finders('graphite.finders.standard.StandardFinder')[0] nodes = list(finder.find_nodes(FindQuery('foo;bar=baz2', None, None))) self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].reader.real_metric_path, 'foo;bar=baz2') finally: self.wipe_whisper()
def test_ceres_finder(self): test_dir = join(settings.CERES_DIR) def create_ceres(metric): if not isdir(test_dir): os.makedirs(test_dir) tree = ceres.CeresTree(test_dir) options = {} tree.createNode(metric, **options) tree.store(metric, [(1, 60)]) def wipe_ceres(): try: shutil.rmtree(test_dir) except OSError: pass self.addCleanup(wipe_ceres) create_ceres('foo') create_ceres('foo.bar.baz') create_ceres('bar.baz.foo') finder = get_finders('graphite.finders.ceres.CeresFinder')[0] nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 1) nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) # No data in the expected time period nodes = finder.find_nodes(FindQuery('foo.bar.baz', 10000, 10060)) self.assertEqual(len(list(nodes)), 0) nodes = finder.find_nodes(FindQuery('foo.bar', None, None)) self.assertEqual(len(list(nodes)), 1) nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) # Search for something that isn't valid Ceres content fh = open(join(test_dir, 'foo', 'blah'), 'wb') fh.close() nodes = finder.find_nodes(FindQuery('foo.blah', None, None)) self.assertEqual(len(list(nodes)), 0) # get index result = finder.get_index({}) self.assertEqual(result, ['bar.baz.foo', 'foo', 'foo.bar.baz'])
def query_test_with_submetrics(query_pattern, fg_data, search_results, bg_data=[]): # query_pattern is the pattern to search for # fg_data -> data returned by the "mainthread" call to find metrics # search_results are the expected results # bg_data - non submetric calls do 2 calls to find_metrics, (one # in a background thread,) # bg_data simulates what gets returnd by the background thread def json_callback(request, context): print("json thread callback" + threading.current_thread().name) return fg_data qlen = len(query_pattern.split(".")) with requests_mock.mock() as m: query = FindQuery(query_pattern, 1, 2) m.get(endpoint_old, json=json_callback, status_code=200) metrics = self.finder.find_nodes(query) self.assertSetEqual(set(map(get_path, list(metrics))), set(map(get_start(qlen), search_results))) enum_vals = ['v1', 'v2'] query_test("*", [{'a': False}, {'b': False}], ['a', 'b']) query_test("a.*", [{'a.b': False}], ['a.b']) query_test("a.b.*", [{'a.b.c': True}], [self.metric1]) query_test("a.b.c", [{'a.b.c': True}], [self.metric1]) query_test("a.b.c.*", [{ 'a.b.c.d': False }, { 'a.b.c.v1': True }, { 'a.b.c.v2': True }], [self.metric1 + '.' + v for v in enum_vals] + [self.metric1 + '.d']) query_test("a.b.*.v*", [{ 'a.b.c.v': False }, { 'a.b.c.v1': True }, { 'a.b.c.v2': True }], [self.metric1 + '.' + v for v in enum_vals] + [self.metric1 + '.v.e']) query_test("a.b.*.v*.*", [{ 'a.b.c.v.e': False }], [self.metric1 + '.v.e.f']) # now again, with submetrics self.finder.enable_submetrics = True query_test_with_submetrics("*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': self.metric2, u'unit': u'percent' }], [self.metric1, self.metric2]) query_test_with_submetrics("a.*", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test_with_submetrics("a.*", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1]) query_test_with_submetrics("a.b.*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': 'a.bb.c', u'unit': u'percent' }], [self.metric1]) query_test_with_submetrics("a.b.c", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test_with_submetrics("a.b.c.*", [ { u'metric': self.metric1, u'unit': u'percent' }, { u'metric': (self.metric1 + 'd'), u'unit': u'percent' } ], [self.metric1 + '.' + k for k in self.finder.submetric_aliases]) query_test_with_submetrics("a.b.c.*", [ { u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }, { u'metric': (self.metric1 + 'd'), u'unit': u'percent' } ], [self.metric1 + '.' + k for k in self.finder.submetric_aliases]) query_test_with_submetrics("a.b.c._avg", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1 + '.' + self.alias_key]) query_test_with_submetrics("a.b.c._avg", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + self.alias_key]) query_test_with_submetrics("a.b.c.v1._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.v1']) query_test_with_submetrics( "a.b.c.*._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + v for v in enum_vals]) query_test_with_submetrics( "a.b.*.*._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + v for v in enum_vals]) query_test_with_submetrics( "a.b.c.v*._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + v for v in enum_vals])
def test_standard_finder(self, scandir_mock): try: self.create_whisper('foo.wsp') self.create_whisper(join('foo', 'bar', 'baz.wsp')) self.create_whisper(join('bar', 'baz', 'foo.wsp')) self.create_whisper( join('_tagged', '9c6', '79b', 'foo;bar=baz.wsp')) self.create_whisper( join( '_tagged', 'b34', '2de', # foo;bar=baz2 'b342defa10cb579981c63ef78be5ac248f681f4bd2c35bc0209d3a7b9eb99346.wsp' )) finder = get_finders('graphite.finders.standard.StandardFinder')[0] scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 4) scandir_mock.call_count = 0 nodes = finder.find_nodes( FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes( FindQuery('foo.{ba{r,z},baz}.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes( FindQuery('{foo,garbage}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes( FindQuery('{fo,ba}{o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes( FindQuery('{fo,ba}{o,o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes( FindQuery('{fo,ba}{o,z}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo;bar=baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo;bar=baz2', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) results = finder.fetch(['foo'], 0, 1) self.assertEqual(results, []) finally: scandir_mock.call_count = 0 self.wipe_whisper()
def run_find(self, finder): nodes = list(finder.find_nodes(FindQuery('rackspace.*', 0, 100))) self.assertTrue(len(nodes) > 0)
def test_standard_finder(self): def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock self.create_whisper('foo.wsp') self.create_whisper(join('foo', 'bar', 'baz.wsp')) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 2) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) finally: os.listdir = self._original_listdir self.wipe_whisper()
def test_finder(self): finder = TenantBluefloodFinder(config) nodes = list(finder.find_nodes(FindQuery('rackspace.*', 0, 100))) self.assertTrue(len(nodes) > 0)
def test_standard_finder(self,scandir_mock): try: self.create_whisper('foo.wsp') self.create_whisper(join('foo', 'bar', 'baz.wsp')) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 5) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 5) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 2) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) finally: scandir_mock.call_count = 0 self.wipe_whisper()
def test_find_nodes(self): endpoint = self.finder.find_metrics_endpoint( self.finder.bf_query_endpoint, self.finder.tenant) # one time through without submetrics self.finder.enable_submetrics = False with requests_mock.mock() as m: #test 401 errors query = FindQuery("*", 1, 2) m.get(endpoint, json={}, status_code=401) metrics = self.finder.find_nodes(query) self.assertTrue(list(metrics) == []) with requests_mock.mock() as m: query = FindQuery("*", 1, 2) m.get(endpoint, json=exc_callback, status_code=401) with self.assertRaises(ValueError): list(self.finder.find_nodes(query)) def get_start(x): return lambda y: '.'.join(y.split('.')[:x]) get_path = lambda x: x.path def query_test(query_pattern, fg_data, search_results, bg_data=[]): # query_pattern is the pattern to search for # fg_data is the data returned by the "mainthread" call to find metrics # search_results are the expected results # bg_data - non submetric calls do 2 calls to find_metrics, (one in a background thread,) # bg_data simulates what gets returnd by the background thread def json_callback(request, context): print("json thread callback" + threading.current_thread().name) full_query = "include_enum_values=true&" + urllib.urlencode({ 'query': query_pattern }).lower() if not self.finder.enable_submetrics: if request.query == full_query: return fg_data else: return bg_data else: return fg_data qlen = len(query_pattern.split(".")) with requests_mock.mock() as m: query = FindQuery(query_pattern, 1, 2) m.get(endpoint, json=json_callback, status_code=200) metrics = self.finder.find_nodes(query) self.assertSetEqual(set(map(get_path, list(metrics))), set(map(get_start(qlen), search_results))) enum_vals = ['v1', 'v2'] query_test("*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': self.metric2, u'unit': u'percent' }], [self.metric1, self.metric2]) query_test("a.*", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test("a.*", [{ u'metric': self.metric1 + 'd', u'unit': u'percent' }], [self.metric1], [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }]) query_test("a.b.*", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test("a.b.c", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test("a.b.c.*", [{ u'metric': self.metric1 + '.d.e', u'unit': u'percent' }], [self.metric1 + '.' + v for v in enum_vals] + [self.metric1 + '.d.e'], [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }]) query_test("a.b.*.v*", [{ u'metric': self.metric1 + '.v.e', u'unit': u'percent' }], [self.metric1 + '.' + v for v in enum_vals] + [self.metric1 + '.v.e'], [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }]) query_test("a.b.*.v*.*", [{ u'metric': self.metric1 + '.v.e.f', u'unit': u'percent' }], [self.metric1 + '.v.e.f'], [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }]) # now again, with submetrics self.finder.enable_submetrics = True query_test("*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': self.metric2, u'unit': u'percent' }], [self.metric1, self.metric2]) query_test("a.*", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test("a.*", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1]) query_test("a.b.*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': 'a.bb.c', u'unit': u'percent' }], [self.metric1]) query_test("a.b.c", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1]) query_test("a.b.c.*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': (self.metric1 + 'd'), u'unit': u'percent' }], [self.metric1 + '.' + k for k in self.finder.submetric_aliases]) query_test("a.b.c.*", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }, { u'metric': (self.metric1 + 'd'), u'unit': u'percent' }], [self.metric1 + '.' + k for k in self.finder.submetric_aliases]) query_test("a.b.c._avg", [{ u'metric': self.metric1, u'unit': u'percent' }], [self.metric1 + '.' + self.alias_key]) query_test("a.b.c._avg", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + self.alias_key]) query_test("a.b.c.v1._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.v1']) query_test("a.b.c.*._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + v for v in enum_vals]) query_test("a.b.*.*._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + v for v in enum_vals]) query_test("a.b.c.v*._enum", [{ u'metric': self.metric1, u'unit': u'percent', u'enum_values': enum_vals }], [self.metric1 + '.' + v for v in enum_vals])
def test_ceres_finder(self): test_dir = join(settings.CERES_DIR) def create_ceres(metric): if not isdir(test_dir): os.makedirs(test_dir) tree = ceres.CeresTree(test_dir) options = {} tree.createNode(metric, **options) tree.store(metric, [(1, 60)]) def wipe_ceres(): try: shutil.rmtree(test_dir) except OSError: pass def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock create_ceres('foo') create_ceres('foo.bar.baz') create_ceres('bar.baz.foo') finder = get_finder('graphite.finders.ceres.CeresFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) # No data in the expected time period self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', 10000, 10060)) self.assertEqual(len(list(nodes)), 0) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 8) # Search for something that isn't valid Ceres content fh = open(join(test_dir, 'foo', 'blah'), 'wb') fh.close() self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.blah', None, None)) self.assertEqual(len(list(nodes)), 0) self.assertEqual(self._listdir_counter, 0) finally: os.listdir = self._original_listdir wipe_ceres()
def test_find_nodes(self): endpoint = self.finder.find_metrics_endpoint( self.finder.bf_query_endpoint, self.finder.tenant) # one time through without submetrics self.finder.enable_submetrics = False with requests_mock.mock() as m: #test 401 errors query = FindQuery("*", 1, 2) m.get(endpoint, json={}, status_code=401) metrics = self.finder.find_nodes(query) self.assertTrue(list(metrics) == []) with requests_mock.mock() as m: query = FindQuery("*", 1, 2) m.get(endpoint, json=exc_callback, status_code=401) with self.assertRaises(ValueError): list(self.finder.find_nodes(query)) def get_start(x): return lambda y: '.'.join(y.split('.')[0:x]) get_path = lambda x: x.path def query_test(query_pattern, jdata, qlen, search_results): with requests_mock.mock() as m: query = FindQuery(query_pattern, 1, 2) m.get(endpoint, json=jdata, status_code=200) metrics = self.finder.find_nodes(query) self.assertSequenceEqual(map(get_path, list(metrics)), map(get_start(qlen), search_results)) query_test("*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': self.metric2, u'unit': u'percent' }], 1, [self.metric1, self.metric2]) query_test("a.*", [{ u'metric': self.metric1, u'unit': u'percent' }], 2, [self.metric1]) query_test("a.b.*", [{ u'metric': self.metric1, u'unit': u'percent' }], 3, [self.metric1]) query_test("a.b.c", [{ u'metric': self.metric1, u'unit': u'percent' }], 3, [self.metric1]) # now again, with submetrics self.finder.enable_submetrics = True query_test("*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': self.metric2, u'unit': u'percent' }], 1, [self.metric1, self.metric2]) query_test("a.*", [{ u'metric': self.metric1, u'unit': u'percent' }], 2, [self.metric1]) query_test("a.b.*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': 'a.bb.c', u'unit': u'percent' }], 3, [self.metric1]) query_test("a.b.c", [{ u'metric': self.metric1, u'unit': u'percent' }], 3, [self.metric1]) query_test("a.b.c.*", [{ u'metric': self.metric1, u'unit': u'percent' }, { u'metric': (self.metric1 + 'd'), u'unit': u'percent' }], 4, [self.metric1 + '.' + self.alias_key]) query_test("a.b.c._avg", [{ u'metric': self.metric1, u'unit': u'percent' }], 4, [self.metric1 + '.' + self.alias_key])