def get_structure(prefix=None, finders=None): """Return a hierarchical ``dict`` of nodes/targets that are found in the ``settings.STORAGE_DIR`` """ if finders is None: finders = [get_finder(finder_path) for finder_path in settings.STORAGE_FINDERS] # Go iteratively through the patern "*.*.*......." # and, after some time, return all available nodes as a hierarchical # structure. # Might be also done just by going through the directory # structure? res = {} if prefix is None: pattern = "*" else: pattern = prefix + ".*" for finder in finders: for node in finder.find_nodes(FindQuery(pattern, None, None)): if isinstance(node, LeafNode): res[node.name] = node elif isinstance(node, BranchNode): tmp_res = get_structure(node.path, finders) # If there are no nodes inside a BrancNode we ignore it if tmp_res: res[node.name] = tmp_res else: msg = "Unknown node type: %s, %s"%(node.path, type(node)) raise Exception(msg) return res
def test_standard_finder(self): def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock self.create_whisper('foo.wsp') self.create_whisper(join('foo', 'bar', 'baz.wsp')) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) finally: os.listdir = self._original_listdir self.wipe_whisper()
def test_standard_finder_gzipped_whisper(self): def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock self.create_whisper('foo.wsp', True) self.create_whisper(join('foo', 'bar', 'baz.wsp'), True) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) finally: os.listdir = self._original_listdir self.wipe_whisper()
def index_json(request): jsonp = request.REQUEST.get('jsonp', False) cluster = request.REQUEST.get('cluster', False) if cluster and len(settings.CLUSTER_SERVERS) > 1: matches = reduce( lambda x, y: list(set(x + y)), \ [json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \ for cluster_server in settings.CLUSTER_SERVERS]) else: matches = [ node for finder in settings.STORAGE_FINDERS for node in get_finder(finder).get_all_nodes() ] return json_response_for(request, matches, jsonp=jsonp)
def test_custom_finder(self): store = Store(finders=[get_finder('tests.test_finders.DummyFinder')]) nodes = list(store.find("foo")) self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].path, 'foo') nodes = list(store.find('bar.*')) self.assertEqual(len(nodes), 10) node = nodes[0] self.assertEqual(node.path.split('.')[0], 'bar') time_info, series = node.fetch(100, 200) self.assertEqual(time_info, (100, 200, 10)) self.assertEqual(len(series), 10)
def test_custom_finder(self): store = Store(finders=[get_finder("tests.test_finders.DummyFinder")]) nodes = list(store.find("foo")) self.assertEqual(len(nodes), 1) self.assertEqual(nodes[0].path, "foo") nodes = list(store.find("bar.*")) self.assertEqual(len(nodes), 10) node = nodes[0] self.assertEqual(node.path.split(".")[0], "bar") time_info, series = node.fetch(100, 200) self.assertEqual(time_info, (100, 200, 10)) self.assertEqual(len(series), 10)
def test_standard_finder(self): test_dir = join(settings.WHISPER_DIR, 'test_standard_finder') def create_whisper(path): path = join(test_dir, path) if not isdir(dirname(path)): os.makedirs(dirname(path)) whisper.create(path, [(1, 60)]) def wipe_whisper(): try: shutil.rmtree(test_dir) except OSError: pass def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock create_whisper('foo.wsp') create_whisper(join('foo', 'bar', 'baz.wsp')) create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes( FindQuery('test_standard_finder.foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes( FindQuery('test_standard_finder.foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes( FindQuery('test_standard_finder.*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) finally: os.listdir = self._original_listdir wipe_whisper()
def test_multiple_globstars(self): self.addCleanup(self.wipe_whisper) finder = get_finder('graphite.finders.standard.StandardFinder') store = Store(finders=[finder]) query = "x.**.x.**.x" hits = ["x.x.x", "x._.x.x", "x.x._.x", "x._.x._.x", "x._._.x.x", "x.x._._.x"] misses = ["x.o.x", "o.x.x", "x.x.o", "o.x.x.x", "x.x.x.o", "o._.x._.x", "x._.o._.x", "x._.x._.o"] for path in hits + misses: file = join(path.replace(".", os.sep)) + ".wsp" self.create_whisper(file) paths = [node.path for node in store.find(query, local=True)] for hit in hits: self.assertIn(hit, paths) for miss in misses: self.assertNotIn(miss, paths)
def test_standard_finder(self): test_dir = join(settings.WHISPER_DIR, "test_standard_finder") def create_whisper(path): path = join(test_dir, path) if not isdir(dirname(path)): os.makedirs(dirname(path)) whisper.create(path, [(1, 60)]) def wipe_whisper(): try: shutil.rmtree(test_dir) except OSError: pass def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock create_whisper("foo.wsp") create_whisper(join("foo", "bar", "baz.wsp")) create_whisper(join("bar", "baz", "foo.wsp")) finder = get_finder("graphite.finders.standard.StandardFinder") self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("test_standard_finder.foo", None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("test_standard_finder.foo.bar.baz", None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("test_standard_finder.*.ba?.{baz,foo}", None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) finally: os.listdir = self._original_listdir wipe_whisper()
def test_standard_finder_gzipped_whisper(self, scandir_mock): try: self.create_whisper('foo.wsp', True) self.create_whisper(join('foo', 'bar', 'baz.wsp'), True) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) finally: scandir_mock.call_count = 0 self.wipe_whisper()
def test_standard_finder(self): def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock self.create_whisper("foo.wsp") self.create_whisper(join("foo", "bar", "baz.wsp")) self.create_whisper(join("bar", "baz", "foo.wsp")) finder = get_finder("graphite.finders.standard.StandardFinder") self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("foo", None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("foo.bar.baz", None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("*.ba?.{baz,foo}", None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("{foo,bar}.{baz,bar}.{baz,foo}", None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery("foo.{ba{r,z},baz}.baz", None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) finally: os.listdir = self._original_listdir self.wipe_whisper()
def get_all_leaf_nodes(finders=None): """Return a ``list`` of all leaf nodes/targets that are found in the ``settings.STORAGE_DIR``""" if finders is None: finders = [get_finder(finder_path) for finder_path in settings.STORAGE_FINDERS] # Go iteratively through the patern "*.*.*......." # and, after some time, return all available nodes. # Might be also done just by going through the directory # structure? res = [] pattern = "*" found = True while found: found = False for finder in finders: for node in finder.find_nodes(FindQuery(pattern, None, None)): found = True if isinstance(node, LeafNode): res.append(node.path) pattern += ".*" return res
def test_ceres_finder(self): test_dir = join(settings.CERES_DIR) def create_ceres(metric): if not isdir(test_dir): os.makedirs(test_dir) tree = ceres.CeresTree(test_dir) options = {} tree.createNode(metric, **options) tree.store(metric, [(1, 60)]) def wipe_ceres(): try: shutil.rmtree(test_dir) except OSError: pass def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock create_ceres('foo') create_ceres('foo.bar.baz') create_ceres('bar.baz.foo') finder = get_finder('graphite.finders.ceres.CeresFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) # No data in the expected time period self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', 10000, 10060)) self.assertEqual(len(list(nodes)), 0) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 8) # Search for something that isn't valid Ceres content fh = open(join(test_dir, 'foo', 'blah'), 'wb') fh.close() self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.blah', None, None)) self.assertEqual(len(list(nodes)), 0) self.assertEqual(self._listdir_counter, 0) finally: os.listdir = self._original_listdir wipe_ceres()
def test_standard_finder(self,scandir_mock): try: self.create_whisper('foo.wsp') self.create_whisper(join('foo', 'bar', 'baz.wsp')) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 0) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 5) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mock.call_count, 5) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 2) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) scandir_mock.call_count = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mock.call_count, 1) finally: scandir_mock.call_count = 0 self.wipe_whisper()
def test_standard_finder(self): def listdir_mock(d): self._listdir_counter += 1 return self._original_listdir(d) try: os.listdir = listdir_mock self.create_whisper('foo.wsp') self.create_whisper(join('foo', 'bar', 'baz.wsp')) self.create_whisper(join('bar', 'baz', 'foo.wsp')) finder = get_finder('graphite.finders.standard.StandardFinder') self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 0) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None)) self.assertEqual(len(list(nodes)), 2) self.assertEqual(self._listdir_counter, 5) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 2) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) self._listdir_counter = 0 nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None)) self.assertEqual(len(list(nodes)), 1) self.assertEqual(self._listdir_counter, 1) finally: os.listdir = self._original_listdir self.wipe_whisper()