def test_sorted(self):
for db in (
('test', 'foo.wsp'),
('test', 'welp.wsp'),
('test', 'baz.wsp'),
):
db_path = os.path.join(WHISPER_DIR, *db)
if not os.path.exists(os.path.dirname(db_path)):
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path, [(1, 60)])
response = self.app.get(self.url,
query_string={
'rawData': '1',
'target': 'test.*'
})
dses = response.data.decode('utf-8').strip().split("\n")
paths = []
for ds in dses:
info, data = ds.strip().split('|', 1)
path, start, stop, step = info.split(',')
paths.append(path)
self.assertEqual(paths, ['test.baz', 'test.foo', 'test.welp'])
def test_gzipped_whisper_finder(self, scandir_mocked):
for db in (
('gzwhisper_finder', 'foo.wsp'),
('gzwhisper_finder', 'foo', 'bar', 'baz.wsp'),
('gzwhisper_finder', 'bar', 'baz', 'baz.wsp'),
):
db_path = os.path.join(WHISPER_DIR, *db)
if not os.path.exists(os.path.dirname(db_path)):
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path, [(1, 60)])
with open(db_path, 'rb') as f_in:
f_out = gzip.open("%s.gz" % db_path, 'wb')
shutil.copyfileobj(f_in, f_out)
f_out.close()
os.remove(db_path)
try:
store = app.config['GRAPHITE']['store']
scandir_mocked.call_count = 0
nodes = store.find('gzwhisper_finder.foo')
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mocked.call_count, 0)
scandir_mocked.call_count = 0
nodes = store.find('gzwhisper_finder.foo{}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
finally:
scandir_mocked.call_count = 0
def test_metrics_index(self):
url = '/metrics/index.json'
response = self.app.get(url)
self.assertJSON(response, [])
self.assertEqual(response.headers['Content-Type'], 'application/json')
response = self.app.get(url, query_string={'jsonp': 'foo'})
self.assertEqual(response.data, b'foo([])')
self.assertEqual(response.headers['Content-Type'], 'text/javascript')
parent = os.path.join(WHISPER_DIR, 'collectd')
os.makedirs(parent)
for metric in ['load', 'memory', 'cpu']:
db = os.path.join(parent, '{0}.wsp'.format(metric))
whisper.create(db, [(1, 60)])
response = self.app.get(url)
self.assertJSON(response, [
u'collectd.cpu',
u'collectd.load',
u'collectd.memory',
])
response = self.app.get(url, query_string={'jsonp': 'bar'})
self.assertEqual(
response.data,
b'bar(["collectd.cpu", "collectd.load", "collectd.memory"])')
def test_jsonp(self):
whisper.create(self.db, [(1, 60)])
start = int(time.time()) - 59
response = self.app.get(self.url,
query_string={
'format': 'json',
'jsonp': 'foo',
'target': 'test'
})
data = response.data.decode('utf-8')
self.assertTrue(data.startswith('foo('))
data = json.loads(data[4:-1])
try:
self.assertEqual(
data, [{
'datapoints': [[None, start + i] for i in range(60)],
'target': 'test'
}])
except AssertionError: # Race condition when time overlaps a second
self.assertEqual(
data, [{
'datapoints': [[None, start + i + 1] for i in range(60)],
'target': 'test'
}])
def create_db(self):
whisper.create(self.db, [(1, 60)])
self.ts = int(time.time())
whisper.update(self.db, 1.0, self.ts - 2)
whisper.update(self.db, 0.5, self.ts - 1)
whisper.update(self.db, 1.5, self.ts)
def _create_dbs(self, ts=None):
ts = ts or int(time.time())
for db in (
('test', 'foo.wsp'),
('test', 'wat', 'welp.wsp'),
('test', 'bar', 'baz.wsp'),
):
db_path = os.path.join(WHISPER_DIR, *db)
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path, [(1, 60)])
whisper.update(db_path, 1, ts)
whisper.update(db_path, 2, ts)
def write_series(self, series, retentions=((1, 180), )):
file_name = os.path.join(
WHISPER_DIR,
'{0}.wsp'.format(series.pathExpression.replace('.', os.sep)))
dir_name = os.path.dirname(file_name)
if not os.path.isdir(dir_name):
os.makedirs(dir_name)
whisper.create(file_name, retentions)
data = []
for index, value in enumerate(series):
if value is None:
continue
data.append((series.start + index * series.step, value))
whisper.update_many(file_name, data)
def test_terminal_globstar(self):
store = app.config['GRAPHITE']['store']
query = "z.**"
hits = ["z._", "z._._", "z._._._"]
misses = ["z", "o._", "o.z._", "o._.z"]
for path in hits + misses:
db_path = os.path.join(WHISPER_DIR, path.replace(".", os.sep))
if not os.path.exists(os.path.dirname(db_path)):
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path + '.wsp', [(1, 60)])
paths = [node.path for node in store.find(query, local=True)]
for hit in hits:
self.assertIn(hit, paths)
for miss in misses:
self.assertNotIn(miss, paths)
def test_raw_data(self):
whisper.create(self.db, [(1, 60)])
response = self.app.get(self.url,
query_string={
'rawData': '1',
'target': 'test'
})
info, data = response.data.decode('utf-8').strip().split('|', 1)
path, start, stop, step = info.split(',')
datapoints = data.split(',')
try:
self.assertEqual(datapoints, ['None'] * 60)
self.assertEqual(int(stop) - int(start), 60)
except AssertionError:
self.assertEqual(datapoints, ['None'] * 59)
self.assertEqual(int(stop) - int(start), 59)
self.assertEqual(path, 'test')
self.assertEqual(int(step), 1)
def test_whisper_finder(self, scandir_mocked):
for db in (
('whisper_finder', 'foo.wsp'),
('whisper_finder', 'foo', 'bar', 'baz.wsp'),
('whisper_finder', 'bar', 'baz', 'baz.wsp'),
):
db_path = os.path.join(WHISPER_DIR, *db)
if not os.path.exists(os.path.dirname(db_path)):
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path, [(1, 60)])
try:
store = app.config['GRAPHITE']['store']
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.foo')
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mocked.call_count, 0)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.foo.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 0)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.*.ba?.{baz,foo}')
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mocked.call_count, 5)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{foo,bar}.{baz,bar}.{baz,foo}')
self.assertEqual(len(list(nodes)), 2)
self.assertEqual(scandir_mocked.call_count, 5)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{foo}.bar.*')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 2)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.foo.{ba{r,z},baz}.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{foo,garbage}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{fo{o}}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.foo{}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{fo,ba}{o}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{fo,ba}{o,o}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
scandir_mocked.call_count = 0
nodes = store.find('whisper_finder.{fo,ba}{o,z}.bar.baz')
self.assertEqual(len(list(nodes)), 1)
self.assertEqual(scandir_mocked.call_count, 1)
finally:
scandir_mocked.call_count = 0
def test_templates(self):
ts = int(time.time())
value = 1
for db in (
('hosts', 'worker1', 'cpu.wsp'),
('hosts', 'worker2', 'cpu.wsp'),
):
db_path = os.path.join(WHISPER_DIR, *db)
if not os.path.exists(os.path.dirname(db_path)):
os.makedirs(os.path.dirname(db_path))
whisper.create(db_path, [(1, 60)])
whisper.update(db_path, value, ts)
value += 1
for query, expected in [
({
'target': 'template(hosts.worker1.cpu)'
}, 'hosts.worker1.cpu'),
({
'target': 'template(constantLine($1),12)'
}, '12'),
({
'target': 'template(constantLine($1))',
'template[1]': '12'
}, '12.0'),
({
'target': 'template(constantLine($num),num=12)'
}, '12'),
({
'target': 'template(constantLine($num))',
'template[num]': '12'
}, '12.0'),
({
'target': 'template(time($1),"nameOfSeries")'
}, 'nameOfSeries'),
({
'target': 'template(time($1))',
'template[1]': 'nameOfSeries'
}, 'nameOfSeries'),
({
'target': 'template(time($name),name="nameOfSeries")'
}, 'nameOfSeries'),
({
'target': 'template(time($name))',
'template[name]': 'nameOfSeries'
}, 'nameOfSeries'),
({
'target': 'template(sumSeries(hosts.$1.cpu),"worker1")'
}, 'sumSeries(hosts.worker1.cpu)'),
({
'target': 'template(sumSeries(hosts.$1.cpu))',
'template[1]': 'worker*'
}, 'sumSeries(hosts.worker*.cpu)'),
({
'target': 'template(sumSeries(hosts.$host.cpu))',
'template[host]': 'worker*'
}, 'sumSeries(hosts.worker*.cpu)'),
]:
query['format'] = 'json'
response = self.app.get(self.url, query_string=query)
data = json.loads(response.data.decode('utf-8'))
self.assertEqual(data[0]['target'], expected)
def test_render_validation(self):
whisper.create(self.db, [(1, 60)])
response = self.app.get(self.url)
self.assertJSON(response,
{'errors': {
'target': 'This parameter is required.'
}},
status_code=400)
response = self.app.get(self.url,
query_string={
'graphType': 'foo',
'target': 'test'
})
render_status_code = 200 if CAIRO_ENABLED else 400
expected_response = {'errors': {
'graphType': "Invalid graphType 'foo', must be one of 'line', "
"'pie'."}} if CAIRO_ENABLED else \
{'errors': {
'format': 'Requested image or pdf format but cairo library '
'is not available'}}
self.assertJSON(response, expected_response, status_code=400)
response = self.app.get(self.url,
query_string={
'maxDataPoints': 'foo',
'target': 'test'
})
self.assertJSON(response,
{'errors': {
'maxDataPoints': 'Must be an integer.'
}},
status_code=400)
response = self.app.get(self.url,
query_string={
'from': '21:2020140313',
'until': '21:2020140313',
'target': 'test'
})
self.assertJSON(response, {
'errors': {
'from': 'Invalid empty time range',
'until': 'Invalid empty time range',
}
},
status_code=400)
response = self.app.get(self.url,
query_string={
'target': 'foo',
'width': 100,
'thickness': '1.5',
'fontBold': 'true',
'fontItalic': 'default',
})
self.assertEqual(response.status_code, render_status_code)
response = self.app.get(self.url,
query_string={
'target': 'foo',
'tz': 'Europe/Lausanne'
})
self.assertJSON(
response,
{'errors': {
'tz': "Unknown timezone: 'Europe/Lausanne'.",
}},
status_code=400)
response = self.app.get(self.url,
query_string={
'target': 'test:aa',
'graphType': 'pie'
})
if CAIRO_ENABLED:
self.assertJSON(
response,
{'errors': {
'target': "Invalid target: 'test:aa'.",
}},
status_code=400)
else:
self.assertJSON(response, self.cairo_missing_resp, status_code=400)
response = self.app.get(self.url,
query_string={
'target': ['test', 'foo:1.2'],
'graphType': 'pie'
})
self.assertEqual(response.status_code, render_status_code)
response = self.app.get(self.url,
query_string={'target': ['test', '']})
self.assertEqual(response.status_code, render_status_code)
response = self.app.get(self.url,
query_string={
'target': 'test',
'format': 'csv'
})
lines = response.data.decode('utf-8').strip().split('\n')
# 59 is a time race cond
self.assertTrue(len(lines) in [59, 60])
self.assertFalse(any([l.strip().split(',')[2] for l in lines]))
response = self.app.get(self.url,
query_string={
'target': 'test',
'format': 'svg',
'jsonp': 'foo'
})
if CAIRO_ENABLED:
jsonpsvg = response.data.decode('utf-8')
self.assertTrue(
jsonpsvg.startswith('foo("<?xml version=\\"1.0\\"'))
self.assertTrue(jsonpsvg.endswith('</script>\\n</svg>")'))
else:
self.assertTrue(response.status_code, render_status_code)
response = self.app.get(self.url,
query_string={
'target': 'test',
'format': 'svg'
})
if CAIRO_ENABLED:
svg = response.data.decode('utf-8')
self.assertTrue(svg.startswith('<?xml version="1.0"'))
else:
self.assertJSON(response, self.cairo_missing_resp, status_code=400)
response = self.app.get(self.url,
query_string={
'target': 'inexisting',
'format': 'svg'
})
if CAIRO_ENABLED:
self.assertEqual(response.status_code, render_status_code)
svg = response.data.decode('utf-8')
self.assertTrue(svg.startswith('<?xml version="1.0"'))
else:
self.assertJSON(response, self.cairo_missing_resp, status_code=400)
response = self.app.get(self.url,
query_string={
'target': 'sum(test)',
})
self.assertEqual(response.status_code, render_status_code)
response = self.app.get(self.url,
query_string={
'target': [
'sinFunction("a test", 2)',
'sinFunction("other test", 2.1)',
'sinFunction("other test", 2e1)'
],
})
self.assertEqual(response.status_code, render_status_code)
response = self.app.get(
self.url,
query_string={
'target': ['percentileOfSeries(sin("foo bar"), 95, true)']
})
self.assertEqual(response.status_code, render_status_code)
def test_render_options(self):
# No rendering to test without cairo, skip if disabled
if not CAIRO_ENABLED:
return
self.create_db()
db2 = os.path.join(WHISPER_DIR, 'foo.wsp')
whisper.create(db2, [(1, 60)])
ts = int(time.time())
whisper.update(db2, 0.5, ts - 2)
for qs in [
{
'logBase': 'e'
},
{
'logBase': 1
},
{
'logBase': 0.5
},
{
'logBase': 10
},
{
'margin': -1
},
{
'colorList': 'orange,green,blue,#0f00f0'
},
{
'bgcolor': 'orange'
},
{
'bgcolor': '000000'
},
{
'bgcolor': '#000000'
},
{
'bgcolor': '123456'
},
{
'bgcolor': '#123456'
},
{
'bgcolor': '#12345678'
},
{
'bgcolor': 'aaabbb'
},
{
'bgcolor': '#aaabbb'
},
{
'bgcolor': '#aaabbbff'
},
{
'fontBold': 'true'
},
{
'title': 'Hellò'
},
{
'title': 'true'
},
{
'vtitle': 'Hellò'
},
{
'title': 'Hellò',
'yAxisSide': 'right'
},
{
'uniqueLegend': 'true',
'_expr': 'secondYAxis({0})'
},
{
'uniqueLegend': 'true',
'vtitleRight': 'foo',
'_expr': 'secondYAxis({0})'
},
{
'rightWidth': '1',
'_expr': 'secondYAxis({0})'
},
{
'rightDashed': '1',
'_expr': 'secondYAxis({0})'
},
{
'rightColor': 'black',
'_expr': 'secondYAxis({0})'
},
{
'leftWidth': '1',
'target': ['secondYAxis(foo)', 'test']
},
{
'leftDashed': '1',
'target': ['secondYAxis(foo)', 'test']
},
{
'leftColor': 'black',
'target': ['secondYAxis(foo)', 'test']
},
{
'width': '10',
'_expr': 'secondYAxis({0})'
},
{
'logBase': 'e',
'target': ['secondYAxis(foo)', 'test']
},
{
'graphOnly': 'true',
'yUnitSystem': 'si'
},
{
'graphOnly': 'true',
'yUnitSystem': 'wat'
},
{
'lineMode': 'staircase'
},
{
'lineMode': 'slope'
},
{
'lineMode': 'slope',
'from': '-1s'
},
{
'lineMode': 'connected'
},
{
'min': 1,
'max': 2,
'thickness': 2,
'yUnitSystem': 'none'
},
{
'yMax': 5,
'yLimit': 0.5,
'yStep': 0.1
},
{
'yMax': 'max',
'yUnitSystem': 'binary'
},
{
'yMaxLeft': 5,
'yLimitLeft': 0.5,
'yStepLeft': 0.1,
'_expr': 'secondYAxis({0})'
},
{
'yMaxRight': 5,
'yLimitRight': 0.5,
'yStepRight': 0.1,
'_expr': 'secondYAxis({0})'
},
{
'yMin': 0,
'yLimit': 0.5,
'yStep': 0.1
},
{
'yMinLeft': 0,
'yLimitLeft': 0.5,
'yStepLeft': 0.1,
'_expr': 'secondYAxis({0})'
},
{
'yMinRight': 0,
'yLimitRight': 0.5,
'yStepRight': 0.1,
'_expr': 'secondYAxis({0})'
},
{
'areaMode': 'stacked',
'_expr': 'stacked({0})'
},
{
'lineMode': 'staircase',
'_expr': 'stacked({0})'
},
{
'areaMode': 'first',
'_expr': 'stacked({0})'
},
{
'areaMode': 'all',
'_expr': 'stacked({0})'
},
{
'areaMode': 'all',
'areaAlpha': 0.5,
'_expr': 'secondYAxis({0})'
},
{
'areaMode': 'all',
'areaAlpha': 0.5,
'target': ['secondYAxis(foo)', 'test']
},
{
'areaMode': 'stacked',
'areaAlpha': 0.5,
'_expr': 'stacked({0})'
},
{
'areaMode': 'stacked',
'areaAlpha': 'a',
'_expr': 'stacked({0})'
},
{
'areaMode': 'stacked',
'_expr': 'drawAsInfinite({0})'
},
{
'_expr': 'dashed(lineWidth({0}, 5))'
},
{
'target': 'areaBetween(*)'
},
{
'drawNullAsZero': 'true'
},
{
'_expr': 'drawAsInfinite({0})'
},
{
'graphType': 'pie',
'pieMode': 'average',
'title': 'Pie'
},
{
'graphType': 'pie',
'pieMode': 'maximum',
'title': 'Pie'
},
{
'graphType': 'pie',
'pieMode': 'minimum',
'title': 'Pie'
},
{
'graphType': 'pie',
'pieMode': 'average',
'hideLegend': 'true'
},
{
'graphType': 'pie',
'pieMode': 'average',
'valueLabels': 'none'
},
{
'graphType': 'pie',
'pieMode': 'average',
'valueLabels': 'number'
},
{
'graphType': 'pie',
'pieMode': 'average',
'pieLabels': 'rotated'
},
{
'graphType': 'pie',
'pieMode': 'average',
'areaAlpha': '0.1'
},
{
'graphType': 'pie',
'pieMode': 'average',
'areaAlpha': 'none'
},
{
'graphType': 'pie',
'pieMode': 'average',
'valueLabelsColor': 'white'
},
{
'noCache': 'true'
},
{
'cacheTimeout': 5
},
{
'cacheTimeout': 5
}, # cache hit
{
'tz': 'Europe/Berlin'
},
]:
if qs.setdefault('target', ['foo', 'test']) == ['foo', 'test']:
if '_expr' in qs:
expr = qs.pop('_expr')
qs['target'] = [expr.format(t) for t in qs['target']]
response = self.app.get(self.url, query_string=qs)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.headers['Content-Type'], 'image/png')
if Cache is None or qs.get('noCache'):
self.assertEqual(response.headers['Pragma'], 'no-cache')
self.assertEqual(response.headers['Cache-Control'], 'no-cache')
self.assertFalse('Expires' in response.headers)
else:
self.assertEqual(
response.headers['Cache-Control'],
'max-age={0}'.format(qs.get('cacheTimeout', 60)))
self.assertNotEqual(response.headers['Cache-Control'],
'no-cache')
self.assertFalse('Pragma' in response.headers)
for qs in [
{
'bgcolor': 'foo'
},
]:
qs['target'] = 'test'
with self.assertRaises(ValueError):
response = self.app.get(self.url, query_string=qs)
for qs in [
{
'lineMode': 'stacked'
},
]:
qs['target'] = 'test'
with self.assertRaises(AssertionError):
response = self.app.get(self.url, query_string=qs)
