def test_sorted(self): for db in ( ('test', 'foo.wsp'), ('test', 'welp.wsp'), ('test', 'baz.wsp'), ): db_path = os.path.join(WHISPER_DIR, *db) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) whisper.create(db_path, [(1, 60)]) response = self.app.get(self.url, query_string={ 'rawData': '1', 'target': 'test.*' }) dses = response.data.decode('utf-8').strip().split("\n") paths = [] for ds in dses: info, data = ds.strip().split('|', 1) path, start, stop, step = info.split(',') paths.append(path) self.assertEqual(paths, ['test.baz', 'test.foo', 'test.welp'])
def test_gzipped_whisper_finder(self, scandir_mocked): for db in ( ('gzwhisper_finder', 'foo.wsp'), ('gzwhisper_finder', 'foo', 'bar', 'baz.wsp'), ('gzwhisper_finder', 'bar', 'baz', 'baz.wsp'), ): db_path = os.path.join(WHISPER_DIR, *db) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) whisper.create(db_path, [(1, 60)]) with open(db_path, 'rb') as f_in: f_out = gzip.open("%s.gz" % db_path, 'wb') shutil.copyfileobj(f_in, f_out) f_out.close() os.remove(db_path) try: store = app.config['GRAPHITE']['store'] scandir_mocked.call_count = 0 nodes = store.find('gzwhisper_finder.foo') self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 0) scandir_mocked.call_count = 0 nodes = store.find('gzwhisper_finder.foo{}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) finally: scandir_mocked.call_count = 0
def test_metrics_index(self): url = '/metrics/index.json' response = self.app.get(url) self.assertJSON(response, []) self.assertEqual(response.headers['Content-Type'], 'application/json') response = self.app.get(url, query_string={'jsonp': 'foo'}) self.assertEqual(response.data, b'foo([])') self.assertEqual(response.headers['Content-Type'], 'text/javascript') parent = os.path.join(WHISPER_DIR, 'collectd') os.makedirs(parent) for metric in ['load', 'memory', 'cpu']: db = os.path.join(parent, '{0}.wsp'.format(metric)) whisper.create(db, [(1, 60)]) response = self.app.get(url) self.assertJSON(response, [ u'collectd.cpu', u'collectd.load', u'collectd.memory', ]) response = self.app.get(url, query_string={'jsonp': 'bar'}) self.assertEqual( response.data, b'bar(["collectd.cpu", "collectd.load", "collectd.memory"])')
def test_jsonp(self): whisper.create(self.db, [(1, 60)]) start = int(time.time()) - 59 response = self.app.get(self.url, query_string={ 'format': 'json', 'jsonp': 'foo', 'target': 'test' }) data = response.data.decode('utf-8') self.assertTrue(data.startswith('foo(')) data = json.loads(data[4:-1]) try: self.assertEqual( data, [{ 'datapoints': [[None, start + i] for i in range(60)], 'target': 'test' }]) except AssertionError: # Race condition when time overlaps a second self.assertEqual( data, [{ 'datapoints': [[None, start + i + 1] for i in range(60)], 'target': 'test' }])
def create_db(self): whisper.create(self.db, [(1, 60)]) self.ts = int(time.time()) whisper.update(self.db, 1.0, self.ts - 2) whisper.update(self.db, 0.5, self.ts - 1) whisper.update(self.db, 1.5, self.ts)
def _create_dbs(self, ts=None): ts = ts or int(time.time()) for db in ( ('test', 'foo.wsp'), ('test', 'wat', 'welp.wsp'), ('test', 'bar', 'baz.wsp'), ): db_path = os.path.join(WHISPER_DIR, *db) os.makedirs(os.path.dirname(db_path)) whisper.create(db_path, [(1, 60)]) whisper.update(db_path, 1, ts) whisper.update(db_path, 2, ts)
def write_series(self, series, retentions=((1, 180), )): file_name = os.path.join( WHISPER_DIR, '{0}.wsp'.format(series.pathExpression.replace('.', os.sep))) dir_name = os.path.dirname(file_name) if not os.path.isdir(dir_name): os.makedirs(dir_name) whisper.create(file_name, retentions) data = [] for index, value in enumerate(series): if value is None: continue data.append((series.start + index * series.step, value)) whisper.update_many(file_name, data)
def test_terminal_globstar(self): store = app.config['GRAPHITE']['store'] query = "z.**" hits = ["z._", "z._._", "z._._._"] misses = ["z", "o._", "o.z._", "o._.z"] for path in hits + misses: db_path = os.path.join(WHISPER_DIR, path.replace(".", os.sep)) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) whisper.create(db_path + '.wsp', [(1, 60)]) paths = [node.path for node in store.find(query, local=True)] for hit in hits: self.assertIn(hit, paths) for miss in misses: self.assertNotIn(miss, paths)
def test_raw_data(self): whisper.create(self.db, [(1, 60)]) response = self.app.get(self.url, query_string={ 'rawData': '1', 'target': 'test' }) info, data = response.data.decode('utf-8').strip().split('|', 1) path, start, stop, step = info.split(',') datapoints = data.split(',') try: self.assertEqual(datapoints, ['None'] * 60) self.assertEqual(int(stop) - int(start), 60) except AssertionError: self.assertEqual(datapoints, ['None'] * 59) self.assertEqual(int(stop) - int(start), 59) self.assertEqual(path, 'test') self.assertEqual(int(step), 1)
def test_whisper_finder(self, scandir_mocked): for db in ( ('whisper_finder', 'foo.wsp'), ('whisper_finder', 'foo', 'bar', 'baz.wsp'), ('whisper_finder', 'bar', 'baz', 'baz.wsp'), ): db_path = os.path.join(WHISPER_DIR, *db) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) whisper.create(db_path, [(1, 60)]) try: store = app.config['GRAPHITE']['store'] scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.foo') self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 0) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.foo.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 0) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.*.ba?.{baz,foo}') self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 5) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{foo,bar}.{baz,bar}.{baz,foo}') self.assertEqual(len(list(nodes)), 2) self.assertEqual(scandir_mocked.call_count, 5) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{foo}.bar.*') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 2) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.foo.{ba{r,z},baz}.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{foo,garbage}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{fo{o}}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.foo{}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{fo,ba}{o}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{fo,ba}{o,o}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) scandir_mocked.call_count = 0 nodes = store.find('whisper_finder.{fo,ba}{o,z}.bar.baz') self.assertEqual(len(list(nodes)), 1) self.assertEqual(scandir_mocked.call_count, 1) finally: scandir_mocked.call_count = 0
def test_templates(self): ts = int(time.time()) value = 1 for db in ( ('hosts', 'worker1', 'cpu.wsp'), ('hosts', 'worker2', 'cpu.wsp'), ): db_path = os.path.join(WHISPER_DIR, *db) if not os.path.exists(os.path.dirname(db_path)): os.makedirs(os.path.dirname(db_path)) whisper.create(db_path, [(1, 60)]) whisper.update(db_path, value, ts) value += 1 for query, expected in [ ({ 'target': 'template(hosts.worker1.cpu)' }, 'hosts.worker1.cpu'), ({ 'target': 'template(constantLine($1),12)' }, '12'), ({ 'target': 'template(constantLine($1))', 'template[1]': '12' }, '12.0'), ({ 'target': 'template(constantLine($num),num=12)' }, '12'), ({ 'target': 'template(constantLine($num))', 'template[num]': '12' }, '12.0'), ({ 'target': 'template(time($1),"nameOfSeries")' }, 'nameOfSeries'), ({ 'target': 'template(time($1))', 'template[1]': 'nameOfSeries' }, 'nameOfSeries'), ({ 'target': 'template(time($name),name="nameOfSeries")' }, 'nameOfSeries'), ({ 'target': 'template(time($name))', 'template[name]': 'nameOfSeries' }, 'nameOfSeries'), ({ 'target': 'template(sumSeries(hosts.$1.cpu),"worker1")' }, 'sumSeries(hosts.worker1.cpu)'), ({ 'target': 'template(sumSeries(hosts.$1.cpu))', 'template[1]': 'worker*' }, 'sumSeries(hosts.worker*.cpu)'), ({ 'target': 'template(sumSeries(hosts.$host.cpu))', 'template[host]': 'worker*' }, 'sumSeries(hosts.worker*.cpu)'), ]: query['format'] = 'json' response = self.app.get(self.url, query_string=query) data = json.loads(response.data.decode('utf-8')) self.assertEqual(data[0]['target'], expected)
def test_render_validation(self): whisper.create(self.db, [(1, 60)]) response = self.app.get(self.url) self.assertJSON(response, {'errors': { 'target': 'This parameter is required.' }}, status_code=400) response = self.app.get(self.url, query_string={ 'graphType': 'foo', 'target': 'test' }) render_status_code = 200 if CAIRO_ENABLED else 400 expected_response = {'errors': { 'graphType': "Invalid graphType 'foo', must be one of 'line', " "'pie'."}} if CAIRO_ENABLED else \ {'errors': { 'format': 'Requested image or pdf format but cairo library ' 'is not available'}} self.assertJSON(response, expected_response, status_code=400) response = self.app.get(self.url, query_string={ 'maxDataPoints': 'foo', 'target': 'test' }) self.assertJSON(response, {'errors': { 'maxDataPoints': 'Must be an integer.' }}, status_code=400) response = self.app.get(self.url, query_string={ 'from': '21:2020140313', 'until': '21:2020140313', 'target': 'test' }) self.assertJSON(response, { 'errors': { 'from': 'Invalid empty time range', 'until': 'Invalid empty time range', } }, status_code=400) response = self.app.get(self.url, query_string={ 'target': 'foo', 'width': 100, 'thickness': '1.5', 'fontBold': 'true', 'fontItalic': 'default', }) self.assertEqual(response.status_code, render_status_code) response = self.app.get(self.url, query_string={ 'target': 'foo', 'tz': 'Europe/Lausanne' }) self.assertJSON( response, {'errors': { 'tz': "Unknown timezone: 'Europe/Lausanne'.", }}, status_code=400) response = self.app.get(self.url, query_string={ 'target': 'test:aa', 'graphType': 'pie' }) if CAIRO_ENABLED: self.assertJSON( response, {'errors': { 'target': "Invalid target: 'test:aa'.", }}, status_code=400) else: self.assertJSON(response, self.cairo_missing_resp, status_code=400) response = self.app.get(self.url, query_string={ 'target': ['test', 'foo:1.2'], 'graphType': 'pie' }) self.assertEqual(response.status_code, render_status_code) response = self.app.get(self.url, query_string={'target': ['test', '']}) self.assertEqual(response.status_code, render_status_code) response = self.app.get(self.url, query_string={ 'target': 'test', 'format': 'csv' }) lines = response.data.decode('utf-8').strip().split('\n') # 59 is a time race cond self.assertTrue(len(lines) in [59, 60]) self.assertFalse(any([l.strip().split(',')[2] for l in lines])) response = self.app.get(self.url, query_string={ 'target': 'test', 'format': 'svg', 'jsonp': 'foo' }) if CAIRO_ENABLED: jsonpsvg = response.data.decode('utf-8') self.assertTrue( jsonpsvg.startswith('foo("<?xml version=\\"1.0\\"')) self.assertTrue(jsonpsvg.endswith('</script>\\n</svg>")')) else: self.assertTrue(response.status_code, render_status_code) response = self.app.get(self.url, query_string={ 'target': 'test', 'format': 'svg' }) if CAIRO_ENABLED: svg = response.data.decode('utf-8') self.assertTrue(svg.startswith('<?xml version="1.0"')) else: self.assertJSON(response, self.cairo_missing_resp, status_code=400) response = self.app.get(self.url, query_string={ 'target': 'inexisting', 'format': 'svg' }) if CAIRO_ENABLED: self.assertEqual(response.status_code, render_status_code) svg = response.data.decode('utf-8') self.assertTrue(svg.startswith('<?xml version="1.0"')) else: self.assertJSON(response, self.cairo_missing_resp, status_code=400) response = self.app.get(self.url, query_string={ 'target': 'sum(test)', }) self.assertEqual(response.status_code, render_status_code) response = self.app.get(self.url, query_string={ 'target': [ 'sinFunction("a test", 2)', 'sinFunction("other test", 2.1)', 'sinFunction("other test", 2e1)' ], }) self.assertEqual(response.status_code, render_status_code) response = self.app.get( self.url, query_string={ 'target': ['percentileOfSeries(sin("foo bar"), 95, true)'] }) self.assertEqual(response.status_code, render_status_code)
def test_render_options(self): # No rendering to test without cairo, skip if disabled if not CAIRO_ENABLED: return self.create_db() db2 = os.path.join(WHISPER_DIR, 'foo.wsp') whisper.create(db2, [(1, 60)]) ts = int(time.time()) whisper.update(db2, 0.5, ts - 2) for qs in [ { 'logBase': 'e' }, { 'logBase': 1 }, { 'logBase': 0.5 }, { 'logBase': 10 }, { 'margin': -1 }, { 'colorList': 'orange,green,blue,#0f00f0' }, { 'bgcolor': 'orange' }, { 'bgcolor': '000000' }, { 'bgcolor': '#000000' }, { 'bgcolor': '123456' }, { 'bgcolor': '#123456' }, { 'bgcolor': '#12345678' }, { 'bgcolor': 'aaabbb' }, { 'bgcolor': '#aaabbb' }, { 'bgcolor': '#aaabbbff' }, { 'fontBold': 'true' }, { 'title': 'Hellò' }, { 'title': 'true' }, { 'vtitle': 'Hellò' }, { 'title': 'Hellò', 'yAxisSide': 'right' }, { 'uniqueLegend': 'true', '_expr': 'secondYAxis({0})' }, { 'uniqueLegend': 'true', 'vtitleRight': 'foo', '_expr': 'secondYAxis({0})' }, { 'rightWidth': '1', '_expr': 'secondYAxis({0})' }, { 'rightDashed': '1', '_expr': 'secondYAxis({0})' }, { 'rightColor': 'black', '_expr': 'secondYAxis({0})' }, { 'leftWidth': '1', 'target': ['secondYAxis(foo)', 'test'] }, { 'leftDashed': '1', 'target': ['secondYAxis(foo)', 'test'] }, { 'leftColor': 'black', 'target': ['secondYAxis(foo)', 'test'] }, { 'width': '10', '_expr': 'secondYAxis({0})' }, { 'logBase': 'e', 'target': ['secondYAxis(foo)', 'test'] }, { 'graphOnly': 'true', 'yUnitSystem': 'si' }, { 'graphOnly': 'true', 'yUnitSystem': 'wat' }, { 'lineMode': 'staircase' }, { 'lineMode': 'slope' }, { 'lineMode': 'slope', 'from': '-1s' }, { 'lineMode': 'connected' }, { 'min': 1, 'max': 2, 'thickness': 2, 'yUnitSystem': 'none' }, { 'yMax': 5, 'yLimit': 0.5, 'yStep': 0.1 }, { 'yMax': 'max', 'yUnitSystem': 'binary' }, { 'yMaxLeft': 5, 'yLimitLeft': 0.5, 'yStepLeft': 0.1, '_expr': 'secondYAxis({0})' }, { 'yMaxRight': 5, 'yLimitRight': 0.5, 'yStepRight': 0.1, '_expr': 'secondYAxis({0})' }, { 'yMin': 0, 'yLimit': 0.5, 'yStep': 0.1 }, { 'yMinLeft': 0, 'yLimitLeft': 0.5, 'yStepLeft': 0.1, '_expr': 'secondYAxis({0})' }, { 'yMinRight': 0, 'yLimitRight': 0.5, 'yStepRight': 0.1, '_expr': 'secondYAxis({0})' }, { 'areaMode': 'stacked', '_expr': 'stacked({0})' }, { 'lineMode': 'staircase', '_expr': 'stacked({0})' }, { 'areaMode': 'first', '_expr': 'stacked({0})' }, { 'areaMode': 'all', '_expr': 'stacked({0})' }, { 'areaMode': 'all', 'areaAlpha': 0.5, '_expr': 'secondYAxis({0})' }, { 'areaMode': 'all', 'areaAlpha': 0.5, 'target': ['secondYAxis(foo)', 'test'] }, { 'areaMode': 'stacked', 'areaAlpha': 0.5, '_expr': 'stacked({0})' }, { 'areaMode': 'stacked', 'areaAlpha': 'a', '_expr': 'stacked({0})' }, { 'areaMode': 'stacked', '_expr': 'drawAsInfinite({0})' }, { '_expr': 'dashed(lineWidth({0}, 5))' }, { 'target': 'areaBetween(*)' }, { 'drawNullAsZero': 'true' }, { '_expr': 'drawAsInfinite({0})' }, { 'graphType': 'pie', 'pieMode': 'average', 'title': 'Pie' }, { 'graphType': 'pie', 'pieMode': 'maximum', 'title': 'Pie' }, { 'graphType': 'pie', 'pieMode': 'minimum', 'title': 'Pie' }, { 'graphType': 'pie', 'pieMode': 'average', 'hideLegend': 'true' }, { 'graphType': 'pie', 'pieMode': 'average', 'valueLabels': 'none' }, { 'graphType': 'pie', 'pieMode': 'average', 'valueLabels': 'number' }, { 'graphType': 'pie', 'pieMode': 'average', 'pieLabels': 'rotated' }, { 'graphType': 'pie', 'pieMode': 'average', 'areaAlpha': '0.1' }, { 'graphType': 'pie', 'pieMode': 'average', 'areaAlpha': 'none' }, { 'graphType': 'pie', 'pieMode': 'average', 'valueLabelsColor': 'white' }, { 'noCache': 'true' }, { 'cacheTimeout': 5 }, { 'cacheTimeout': 5 }, # cache hit { 'tz': 'Europe/Berlin' }, ]: if qs.setdefault('target', ['foo', 'test']) == ['foo', 'test']: if '_expr' in qs: expr = qs.pop('_expr') qs['target'] = [expr.format(t) for t in qs['target']] response = self.app.get(self.url, query_string=qs) self.assertEqual(response.status_code, 200) self.assertEqual(response.headers['Content-Type'], 'image/png') if Cache is None or qs.get('noCache'): self.assertEqual(response.headers['Pragma'], 'no-cache') self.assertEqual(response.headers['Cache-Control'], 'no-cache') self.assertFalse('Expires' in response.headers) else: self.assertEqual( response.headers['Cache-Control'], 'max-age={0}'.format(qs.get('cacheTimeout', 60))) self.assertNotEqual(response.headers['Cache-Control'], 'no-cache') self.assertFalse('Pragma' in response.headers) for qs in [ { 'bgcolor': 'foo' }, ]: qs['target'] = 'test' with self.assertRaises(ValueError): response = self.app.get(self.url, query_string=qs) for qs in [ { 'lineMode': 'stacked' }, ]: qs['target'] = 'test' with self.assertRaises(AssertionError): response = self.app.get(self.url, query_string=qs)