def get_structure(prefix=None, finders=None):
"""Return a hierarchical ``dict`` of nodes/targets that are found in the
``settings.STORAGE_DIR``
"""
if finders is None:
finders = [get_finder(finder_path)
for finder_path in settings.STORAGE_FINDERS]
# Go iteratively through the patern "*.*.*......."
# and, after some time, return all available nodes as a hierarchical
# structure.
# Might be also done just by going through the directory
# structure?
res = {}
if prefix is None:
pattern = "*"
else:
pattern = prefix + ".*"
for finder in finders:
for node in finder.find_nodes(FindQuery(pattern, None, None)):
if isinstance(node, LeafNode):
res[node.name] = node
elif isinstance(node, BranchNode):
tmp_res = get_structure(node.path, finders)
# If there are no nodes inside a BrancNode we ignore it
if tmp_res:
res[node.name] = tmp_res
else:
msg = "Unknown node type: %s, %s"%(node.path, type(node))
raise Exception(msg)
return res
def test_standard_finder(self):
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
self.create_whisper('foo.wsp')
self.create_whisper(join('foo', 'bar', 'baz.wsp'))
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
finally:
os.listdir = self._original_listdir
self.wipe_whisper()
def test_standard_finder_gzipped_whisper(self):
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
self.create_whisper('foo.wsp', True)
self.create_whisper(join('foo', 'bar', 'baz.wsp'), True)
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
finally:
os.listdir = self._original_listdir
self.wipe_whisper()
def index_json(request):
jsonp = request.REQUEST.get('jsonp', False)
cluster = request.REQUEST.get('cluster', False)
if cluster and len(settings.CLUSTER_SERVERS) > 1:
matches = reduce( lambda x, y: list(set(x + y)), \
[json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \
for cluster_server in settings.CLUSTER_SERVERS])
else:
matches = [ node for finder in settings.STORAGE_FINDERS for node in get_finder(finder).get_all_nodes() ]
return json_response_for(request, matches, jsonp=jsonp)
def test_custom_finder(self):
store = Store(finders=[get_finder('tests.test_finders.DummyFinder')])
nodes = list(store.find("foo"))
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].path, 'foo')
nodes = list(store.find('bar.*'))
self.assertEqual(len(nodes), 10)
node = nodes[0]
self.assertEqual(node.path.split('.')[0], 'bar')
time_info, series = node.fetch(100, 200)
self.assertEqual(time_info, (100, 200, 10))
self.assertEqual(len(series), 10)
def test_custom_finder(self):
store = Store(finders=[get_finder("tests.test_finders.DummyFinder")])
nodes = list(store.find("foo"))
self.assertEqual(len(nodes), 1)
self.assertEqual(nodes[0].path, "foo")
nodes = list(store.find("bar.*"))
self.assertEqual(len(nodes), 10)
node = nodes[0]
self.assertEqual(node.path.split(".")[0], "bar")
time_info, series = node.fetch(100, 200)
self.assertEqual(time_info, (100, 200, 10))
self.assertEqual(len(series), 10)
def index_json(request):
jsonp = request.REQUEST.get('jsonp', False)
cluster = request.REQUEST.get('cluster', False)
if cluster and len(settings.CLUSTER_SERVERS) > 1:
matches = reduce( lambda x, y: list(set(x + y)), \
[json.loads(urlopen("http://" + cluster_server + "/metrics/index.json").read()) \
for cluster_server in settings.CLUSTER_SERVERS])
else:
matches = [
node for finder in settings.STORAGE_FINDERS
for node in get_finder(finder).get_all_nodes()
]
return json_response_for(request, matches, jsonp=jsonp)
def test_standard_finder(self):
test_dir = join(settings.WHISPER_DIR, 'test_standard_finder')
def create_whisper(path):
path = join(test_dir, path)
if not isdir(dirname(path)):
os.makedirs(dirname(path))
whisper.create(path, [(1, 60)])
def wipe_whisper():
try:
shutil.rmtree(test_dir)
except OSError:
pass
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
create_whisper('foo.wsp')
create_whisper(join('foo', 'bar', 'baz.wsp'))
create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(
FindQuery('test_standard_finder.foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(
FindQuery('test_standard_finder.foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(
FindQuery('test_standard_finder.*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
finally:
os.listdir = self._original_listdir
wipe_whisper()
def test_multiple_globstars(self):
self.addCleanup(self.wipe_whisper)
finder = get_finder('graphite.finders.standard.StandardFinder')
store = Store(finders=[finder])
query = "x.**.x.**.x"
hits = ["x.x.x", "x._.x.x", "x.x._.x", "x._.x._.x", "x._._.x.x", "x.x._._.x"]
misses = ["x.o.x", "o.x.x", "x.x.o", "o.x.x.x", "x.x.x.o", "o._.x._.x", "x._.o._.x", "x._.x._.o"]
for path in hits + misses:
file = join(path.replace(".", os.sep)) + ".wsp"
self.create_whisper(file)
paths = [node.path for node in store.find(query, local=True)]
for hit in hits:
self.assertIn(hit, paths)
for miss in misses:
self.assertNotIn(miss, paths)
def test_standard_finder(self):
test_dir = join(settings.WHISPER_DIR, "test_standard_finder")
def create_whisper(path):
path = join(test_dir, path)
if not isdir(dirname(path)):
os.makedirs(dirname(path))
whisper.create(path, [(1, 60)])
def wipe_whisper():
try:
shutil.rmtree(test_dir)
except OSError:
pass
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
create_whisper("foo.wsp")
create_whisper(join("foo", "bar", "baz.wsp"))
create_whisper(join("bar", "baz", "foo.wsp"))
finder = get_finder("graphite.finders.standard.StandardFinder")
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("test_standard_finder.foo", None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("test_standard_finder.foo.bar.baz", None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("test_standard_finder.*.ba?.{baz,foo}", None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
finally:
os.listdir = self._original_listdir
wipe_whisper()
def test_standard_finder_gzipped_whisper(self, scandir_mock):
try:
self.create_whisper('foo.wsp', True)
self.create_whisper(join('foo', 'bar', 'baz.wsp'), True)
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
finally:
scandir_mock.call_count = 0
self.wipe_whisper()
def test_standard_finder(self):
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
self.create_whisper("foo.wsp")
self.create_whisper(join("foo", "bar", "baz.wsp"))
self.create_whisper(join("bar", "baz", "foo.wsp"))
finder = get_finder("graphite.finders.standard.StandardFinder")
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("foo", None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("foo.bar.baz", None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("*.ba?.{baz,foo}", None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("{foo,bar}.{baz,bar}.{baz,foo}", None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery("foo.{ba{r,z},baz}.baz", None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
finally:
os.listdir = self._original_listdir
self.wipe_whisper()
def get_all_leaf_nodes(finders=None):
"""Return a ``list`` of all leaf nodes/targets that are found in the
``settings.STORAGE_DIR``"""
if finders is None:
finders = [get_finder(finder_path)
for finder_path in settings.STORAGE_FINDERS]
# Go iteratively through the patern "*.*.*......."
# and, after some time, return all available nodes.
# Might be also done just by going through the directory
# structure?
res = []
pattern = "*"
found = True
while found:
found = False
for finder in finders:
for node in finder.find_nodes(FindQuery(pattern, None, None)):
found = True
if isinstance(node, LeafNode):
res.append(node.path)
pattern += ".*"
return res
def test_ceres_finder(self):
test_dir = join(settings.CERES_DIR)
def create_ceres(metric):
if not isdir(test_dir):
os.makedirs(test_dir)
tree = ceres.CeresTree(test_dir)
options = {}
tree.createNode(metric, **options)
tree.store(metric, [(1, 60)])
def wipe_ceres():
try:
shutil.rmtree(test_dir)
except OSError:
pass
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
create_ceres('foo')
create_ceres('foo.bar.baz')
create_ceres('bar.baz.foo')
finder = get_finder('graphite.finders.ceres.CeresFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
# No data in the expected time period
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', 10000, 10060))
self.assertEqual(len(list(nodes)), 0)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 8)
# Search for something that isn't valid Ceres content
fh = open(join(test_dir, 'foo', 'blah'), 'wb')
fh.close()
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.blah', None, None))
self.assertEqual(len(list(nodes)), 0)
self.assertEqual(self._listdir_counter, 0)
finally:
os.listdir = self._original_listdir
wipe_ceres()
def test_standard_finder(self,scandir_mock):
try:
self.create_whisper('foo.wsp')
self.create_whisper(join('foo', 'bar', 'baz.wsp'))
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 0)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 5)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mock.call_count, 5)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 2)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
scandir_mock.call_count = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mock.call_count, 1)
finally:
scandir_mock.call_count = 0
self.wipe_whisper()
def test_standard_finder(self):
def listdir_mock(d):
self._listdir_counter += 1
return self._original_listdir(d)
try:
os.listdir = listdir_mock
self.create_whisper('foo.wsp')
self.create_whisper(join('foo', 'bar', 'baz.wsp'))
self.create_whisper(join('bar', 'baz', 'foo.wsp'))
finder = get_finder('graphite.finders.standard.StandardFinder')
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 0)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('*.ba?.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{foo,bar}.{baz,bar}.{baz,foo}', None, None))
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(self._listdir_counter, 5)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{foo}.bar.*', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 2)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo.{ba{r,z},baz}.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{foo,garbage}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo{o}}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('foo{}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,o}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
self._listdir_counter = 0
nodes = finder.find_nodes(FindQuery('{fo,ba}{o,z}.bar.baz', None, None))
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(self._listdir_counter, 1)
finally:
os.listdir = self._original_listdir
self.wipe_whisper()
