def test_template_measurement_first(self): del self.finder template = "measurement.host.resource" self.config['influxdb']['templates'] = [template] measurements = ['load', 'idle', 'usage', 'user'] tags = {'host': 'my_host', 'resource': 'cpu', } fields = {'value': 1} self.write_data(measurements, tags, fields) self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('*') nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = sorted(measurements) self.assertEqual(nodes, expected, msg="Got root branch query result %s - wanted %s" % ( nodes, expected,)) query = Query('%s.*' % (measurements[0])) nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = [tags['host']] self.assertEqual(nodes, expected, msg="Got query %s result %s - wanted %s" % ( query.pattern, nodes, expected,)) query = Query('%s.%s.*' % (measurements[0], tags['host'],)) nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = sorted([tags['resource']]) self.assertEqual(nodes, expected, msg="Got query %s result %s - wanted %s" % ( query.pattern, nodes, expected,))
def test_template_drop_path_part(self): del self.finder # Filter out first part of metric, keep the remainder as # measurement name template = "..measurement*" self.config['influxdb']['templates'] = [self.template, template] measurements = ['my_host.cpu.load', 'my_host.cpu.idle', 'my_host.cpu.usage', 'my_host.cpu.user'] fields = {"value": 1} self.write_data(measurements, {}, fields) self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('*') nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = sorted(['my_host', self.metric_prefix]) self.assertEqual(nodes, expected, msg="Got root branch query result %s - wanted %s" % ( nodes, expected,)) split_measurement = measurements[0].split('.') query = Query('%s.*' % (split_measurement[0])) nodes = [n.name for n in self.finder.find_nodes(query)] expected = [split_measurement[1]] self.assertEqual(nodes, expected, msg="Got sub branch query result %s - wanted %s" % ( nodes, expected,)) query = Query('%s.%s.*' % (split_measurement[0], split_measurement[1],)) nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = sorted([m.split('.')[2] for m in measurements]) self.assertEqual(nodes, expected, msg="Got sub branch query result %s - wanted %s" % ( nodes, expected,))
def test_field_template_with_value_field(self): template = "env.host.measurement.field*" del self.finder measurements = ['cpuusage'] fields = {'value': 1} tags = {'host': 'my_host1', 'env': 'my_env1', } metrics = ['.'.join([tags['env'], tags['host'], m]) for m in measurements] self.client.drop_database(self.db_name) self.client.create_database(self.db_name) self.write_data(measurements, tags, fields) self.config['influxdb']['templates'] = [template] self.finder = influxgraph.InfluxDBFinder(self.config) cpu_nodes = list(self.finder.find_nodes(Query('my_env1.my_host1.*'))) expected = measurements self.assertEqual([n.name for n in cpu_nodes], expected) nodes = list(self.finder.find_nodes(Query('my_env1.my_host1.*.*'))) self.assertEqual(nodes, []) _, data = self.finder.fetch_multi(cpu_nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) for metric in metrics: datapoints = [v for v in data[metric] if v] self.assertTrue(len(datapoints) == self.num_datapoints)
def test_find_branch_nodes(self): """Test finding branch nodes by wildcard""" prefix = 'branch_test_prefix' written_branches = [ 'branch_node1.sub_branch1.sub_branch2.sub_branch3', 'branch_node2.sub_branch11.sub_branch22.sub_branch33' ] leaf_nodes = ['leaf_node1', 'leaf_node2'] written_series = [ ".".join([ prefix, branch, leaf_node, ]) for branch in written_branches for leaf_node in leaf_nodes ] data = [{ "measurement": series, "tags": {}, "time": _time, "fields": { "value": 1, } } for series in written_series for _time in [ (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"), ]] self.assertTrue(self.client.write_points(data)) self.finder.build_index() query = Query(prefix + '.*') nodes = list(self.finder.find_nodes(query)) expected = sorted([b.split('.')[0] for b in written_branches]) branches = sorted([n.name for n in nodes]) self.assertEqual(branches, expected, msg="Got branches %s - wanted %s" % ( branches, expected, )) query = Query(prefix + '.branch_node*.*') nodes = list(self.finder.find_nodes(query)) expected = sorted([b.split('.')[1] for b in written_branches]) branches = sorted([n.name for n in nodes]) self.assertEqual(branches, expected, msg="Got branches %s - wanted %s" % ( branches, expected, )) query = Query(prefix + '.branch_node*.sub_branch*.*') nodes = list(self.finder.find_nodes(query)) expected = sorted([b.split('.')[2] for b in written_branches]) branches = sorted([n.name for n in nodes]) self.assertEqual(branches, expected, msg="Got branches list %s - wanted %s" % ( branches, expected, ))
def test_multi_fetch_data_multi_series_configured_aggregation_functions(self): """Test fetching data for multiple series with aggregation functions configured""" nodes = list(self.finder.find_nodes(Query(self.metric_prefix + ".agg_path.*"))) paths = [node.path for node in nodes] aggregation_funcs = sorted(list(set(influxgraph.utils.get_aggregation_func( path, self.finder.aggregation_functions) for path in paths))) expected = sorted(DEFAULT_AGGREGATIONS.values()) self.assertEqual(expected, aggregation_funcs, msg="Expected aggregation functions %s for paths %s - got %s" % ( expected, paths, aggregation_funcs)) time_info, data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue(nodes[0].path in data, msg="Did not get data for requested series %s - got data for %s" % ( nodes[0].path, data.keys(),)) for suffix in ['min', 'max', 'last', 'sum']: series = self.metric_prefix + ".agg_path.%s" % (suffix,) nodes = list(self.finder.find_nodes(Query(series))) time_info, data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue(series in data, msg="Did not get data for requested series %s - got data for %s" % ( series, data.keys(),))
def test_data_with_fields(self): del self.finder template = "host.measurement.field*" self.config['influxdb']['templates'] = [template] measurements = ['cpu-0', 'cpu-1', 'cpu-2', 'cpu-3'] fields = {'load': 1, 'idle': 1, 'usage': 1, 'user': 1, 'user.io': 1, 'idle.io': 1, 'load.io': 1, 'usage.io': 1, } tags = {'host': 'my_host', 'env': 'my_env', } cpu_metrics = ['.'.join(['my_host', m, f]) for m in measurements for f in ['load', 'usage', 'user', 'idle']] io_metrics = ['.'.join(['my_host', m, f]) for m in measurements for f in ['user.io', 'idle.io', 'load.io', 'usage.io']] self.client.drop_database(self.db_name) self.client.create_database(self.db_name) self.write_data(measurements, tags, fields) self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('%s.*' % (tags['host'])) nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = measurements self.assertEqual(nodes, expected, msg="Got query %s result %s - wanted %s" % ( query.pattern, nodes, expected,)) query = Query('%s.*.*' % (tags['host'],)) nodes = list(self.finder.find_nodes(query)) node_paths = sorted([n.path for n in nodes]) expected = sorted(cpu_metrics) self.assertEqual(node_paths, expected, msg="Got query %s result %s - wanted %s" % ( query.pattern, node_paths, expected,)) time_info, data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) for metric in cpu_metrics: self.assertTrue(metric in data, msg="Did not get data for requested series %s - got data for %s" % ( metric, data.keys(),)) datapoints = [v for v in data[metric] if v] self.assertTrue(len(datapoints) == self.num_datapoints, msg="Expected %s datapoints for %s - got %s" % ( self.num_datapoints, metric, len(datapoints),)) time_info, _data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) for metric in cpu_metrics: datapoints = [v for v in _data[metric] if v] self.assertTrue(len(datapoints) == self.num_datapoints, msg="Expected %s datapoints for %s - got %s" % ( self.num_datapoints, metric, len(datapoints),))
def test_find_leaf_nodes(self): """Test finding leaf nodes by wildcard""" prefix = 'branch_test_prefix' written_branches = [ 'branch_node1.sub_branch1.sub_branch2.sub_branch3', 'branch_node2.sub_branch11.sub_branch22.sub_branch33' ] leaf_nodes = ['leaf_node1', 'leaf_node2'] written_series = [ ".".join([ prefix, branch, leaf_node, ]) for branch in written_branches for leaf_node in leaf_nodes ] data = [{ "measurement": series, "tags": {}, "time": _time, "fields": { "value": 1, } } for series in written_series for _time in [ (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"), ]] self.assertTrue(self.client.write_points(data)) self.finder.build_index() query = Query(".".join([ prefix, "branch_node*", "sub_branch*", "sub_branch*", "sub_branch*", "leaf*" ])) nodes = list(self.finder.find_nodes(query)) expected = sorted(leaf_nodes + leaf_nodes) found_leaves = sorted([n.name for n in nodes]) self.assertEqual(found_leaves, expected, msg="Expected leaf node list '%s' - got %s" % (expected, found_leaves)) for node in nodes: self.assertTrue(node.is_leaf, msg="Leaf node %s is not marked as leaf node" % (node)) nodes = [ node.name for node in self.finder.find_nodes(Query("fakeyfakeyfakefake.*")) ] self.assertEqual(nodes, [], msg="Expected empty leaf node list - got %s" % (nodes, ))
def test_get_series_pagination(self): query, limit = Query('*'), 5 series = self.finder.get_all_series( query, limit=limit) self.assertTrue(len(series) == len(self.series), msg="Did not get data for all series with page limit %s" % ( limit,)) query, limit = Query('*'), 10 series = self.finder.get_all_series( query, limit=limit) self.assertTrue(len(series) == len(self.series), msg="Did not get data for all series with page limit %s" % ( limit,))
def test_multi_fetch_non_existant_series(self): """Test single fetch data for a series by name""" path1, path2 = 'fake_path1', 'fake_path2' reader1 = influxgraph.InfluxDBReader( InfluxDBClient(database=self.db_name), path1) reader2 = influxgraph.InfluxDBReader( InfluxDBClient(database=self.db_name), path2) nodes = [ influxgraph.classes.leaf.InfluxDBLeafNode(path1, reader1), influxgraph.classes.leaf.InfluxDBLeafNode(path2, reader2) ] time_info, data = self.finder.fetch_multi( nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) for metric_name in data: self.assertFalse( data[metric_name], msg="Expected no data for non-existant series %s - got %s" % ( metric_name, data, )) fake_nodes = list(self.finder.find_nodes(Query('fake_pathy_path'))) time_info, data = self.finder.fetch_multi( fake_nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertFalse(data)
def test_multi_fetch_data_multi_series(self): """Test fetching data for multiple series by name""" nodes = list( self.finder.find_nodes(Query(self.metric_prefix + ".leaf*"))) time_info, data = self.finder.fetch_multi( nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue( self.series1 in data and self.series2 in data, msg= "Did not get data for requested series %s and %s - got data for %s" % ( self.series1, self.series2, data.keys(), )) self.assertEqual( time_info, (int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s")), self.step), msg="Time info and step do not match our requested values") for i, series in enumerate([self.series1, self.series2]): self.assertTrue(self.steps == len(data[series]), msg="Expected %s datapoints, got %s instead" % ( self.steps, len(data[series]), )) datapoints = [v for v in data[series] if v] self.assertTrue( len(datapoints) == self.num_datapoints, msg="Expected %s datapoints for series %s - got %s" % ( self.num_datapoints, series, len(datapoints), )) self.assertTrue(datapoints[-1] == self.series_values[i])
def test_single_fetch_memcache_integration(self): self.config['influxdb']['memcache'] = {'host': 'localhost'} del self.config['search_index'] self.finder = influxgraph.InfluxDBFinder(self.config) node = list(self.finder.find_nodes(Query(self.series1)))[0] aggregation_func = get_aggregation_func( node.path, self.finder.aggregation_functions) memcache_key = gen_memcache_key(int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s")), aggregation_func, [node.path]) self.finder.memcache.delete(memcache_key) node.reader.fetch(int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue( node.reader.memcache.get(memcache_key), msg="Expected data for %s to be in memcache after a fetch" % (node.path, )) time_info, data = node.reader.fetch( int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) datapoints = [v for v in data if v] self.assertTrue(self.steps == len(data), msg="Expected %s datapoints, got %s instead" % ( self.steps, len(data), ))
def test_configured_deltas(self): del self.finder config = { 'influxdb': { 'host' : 'localhost', 'port' : 8086, 'user' : 'root', 'pass' : 'root', 'db' : self.db_name, # Set data interval to 1 second for queries # of one hour or less 'deltas' : {3600: 1},}, # 'search_index': 'index', } finder = influxgraph.InfluxDBFinder(config) self.assertTrue(finder.deltas) nodes = list(finder.find_nodes(Query(self.series1))) paths = [node.path for node in nodes] time_info, data = finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue(self.series1 in data, msg="Did not get data for requested series %s - got data for %s" % ( self.series1, data.keys(),)) self.assertTrue(len(data[self.series1]) == 3601, msg="Expected exactly %s data points - got %s instead" % ( 3601, len(data[self.series1])))
def test_fill_param_config(self): self.config['influxdb']['fill'] = 0 self.finder = influxgraph.InfluxDBFinder(self.config) self.assertEqual(self.finder.fill_param, self.config['influxdb']['fill']) nodes = list(self.finder.find_nodes(Query(self.series1))) time_info, data = self.finder.fetch_multi( nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue( self.series1 in data, msg="Did not get data for requested series %s - got data for %s" % ( self.series1, data.keys(), )) self.assertEqual( time_info, (int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s")), self.step), msg="Time info and step do not match our requested values") self.assertTrue(len(data[self.series1]) == self.all_datapoints_num, msg="Expected %s datapoints - got %s" % ( self.all_datapoints_num, len(data[self.series1]), )) self.assertTrue( data[self.series1][-1] == self.config['influxdb']['fill'])
def test_find_series_glob_expansion(self): """Test finding metric prefix by glob expansion""" query = Query('{%s}' % (self.metric_prefix)) nodes = [node.name for node in self.finder.find_nodes(query)] self.assertTrue(self.metric_prefix in nodes, msg="Node list does not contain prefix '%s' - %s" % ( self.metric_prefix, nodes))
def test_get_all_series(self): """ """ query = Query('*') series = self.finder.get_all_series(cache=True, limit=1) self.assertTrue(len(series) == len(self.series), msg="Got series list %s for root branch query - expected %s" % ( series, self.series,))
def test_index_load_from_file(self): values = [['carbon.relays.host.dispatcher1.wallTime_us'], ['carbon.relays.host.metricsReceived'], ['carbon.relays.host.metricsDropped'], ['carbon.relays.host.metricsQueued'], ] data = {'results': [{'series': [{ 'columns': ['key'], 'values': values, }]}]} _tempfile = tempfile.NamedTemporaryFile(mode='wt', delete=False) try: _tempfile.write(json.dumps(data)) except Exception: os.unlink(_tempfile.name) raise else: _tempfile.close() expected = ['carbon'] try: self.finder.index.clear() self.finder.build_index(data=self.finder._read_static_data(_tempfile.name)) self.assertEqual([n.name for n in self.finder.find_nodes(Query('*'))], expected) finally: os.unlink(_tempfile.name)
def test_find_series(self): """Test finding a series by name""" nodes = [node.name for node in self.finder.find_nodes(Query(self.series1)) if node.is_leaf] expected = [self.nodes[0]] self.assertEqual(nodes, expected, msg="Got node list %s - wanted %s" % (nodes, expected,))
def test_find_branch(self): """Test getting branch of metric path""" query = Query('fakeyfakeyfakefake') branches = list(self.finder.find_nodes(query)) self.assertEqual(branches, [], msg="Got branches list %s - wanted empty list" % (branches, )) query = Query('*') prefix = 'branch_test_prefix' written_branches = ['branch_node1', 'branch_node2'] leaf_nodes = ['leaf_node1', 'leaf_node2'] written_series = [ ".".join([ prefix, branch, leaf_node, ]) for branch in written_branches for leaf_node in leaf_nodes ] data = [{ "measurement": series, "tags": {}, "time": _time, "fields": { "value": 1, } } for series in written_series for _time in [ (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"), ]] self.assertTrue(self.client.write_points(data)) self.finder.build_index() query = Query(prefix + '.*') # Test getting leaf nodes with wildcard query = Query(prefix + '.branch_node*.*') _nodes = list(self.finder.find_nodes(query)) nodes = sorted([n.path for n in _nodes]) expected = sorted(written_series) self.assertEqual(nodes, expected, msg="Got node list %s - wanted %s" % ( nodes, expected, ))
def test_named_branch_query(self): query = Query(self.metric_prefix) nodes = list(self.finder.find_nodes(query)) node_names = [n.name for n in nodes] self.assertEqual(node_names, [self.metric_prefix], msg="Expected node names %s, got %s" % ( [self.metric_prefix], node_names,)) self.assertFalse(nodes[0].is_leaf, msg="Root branch node incorrectly marked as leaf node")
def test_templated_index_find(self): query = Query('*') nodes = [n.name for n in self.finder.find_nodes(query)] expected = [self.metric_prefix] self.assertEqual(nodes, expected, msg="Got root branch query result %s - wanted %s" % ( nodes, expected,)) query = Query("%s.*" % (self.metric_prefix,)) nodes = [n.name for n in self.finder.find_nodes(query)] expected = [self.tags[self.paths[1]]] self.assertEqual(nodes, expected, msg="Got sub branch query result %s - wanted %s" % ( nodes, expected,)) query = Query("%s.%s.*" % (self.metric_prefix, self.tags[self.paths[1]])) nodes = sorted([n.name for n in self.finder.find_nodes(query)]) expected = sorted(self.measurements) self.assertEqual(nodes, expected, msg="Got sub branch query result %s - wanted %s" % ( nodes, expected,))
def test_multi_tag_values_multi_measurements(self): measurements = ['cpu-0', 'cpu-1', 'cpu-2', 'cpu-3'] fields = {'load': 1, 'idle': 1, 'usage': 1, 'user': 1, } tags = {'host': 'my_host1', 'env': 'my_env1', } data = [{ "measurement": measurement, "tags": tags, "time": _time, "fields": fields, } for measurement in measurements for _time in [ (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"), ]] metrics = ['.'.join([tags['host'], m, f]) for f in fields.keys() for m in measurements] self.client.drop_database(self.db_name) self.client.create_database(self.db_name) self.assertTrue(self.client.write_points(data)) tags_env2 = {'host': 'my_host1', 'env': 'my_env2', } for d in data: d['tags'] = tags_env2 self.assertTrue(self.client.write_points(data)) template = "env.host.measurement.field*" self.config['influxdb']['templates'] = [template] self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('*.*.*.*') nodes = list(self.finder.find_nodes(query)) node_paths = sorted([n.path for n in nodes]) tag_values = set(['.'.join([t['env'], t['host']]) for t in [tags, tags_env2]]) _metrics = ['.'.join([t, m, f]) for t in tag_values for f in fields.keys() if not '.' in f for m in measurements] expected = sorted(_metrics) self.assertEqual(node_paths, expected, msg="Expected %s nodes - got %s" % ( len(expected), len(node_paths))) _, multi_tag_data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) for metric in _metrics: datapoints = [v for v in multi_tag_data[metric] if v] self.assertTrue(len(datapoints) == self.num_datapoints, msg="Expected %s datapoints for %s - got %s" % ( self.num_datapoints, metric, len(datapoints),))
def test_tagged_data_no_template_config(self): del self.finder self.config['influxdb']['templates'] = None self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('*') nodes = sorted([n.name for n in self.finder.find_nodes(query)]) # expected = [self.metric_prefix] expected = sorted(self.measurements) self.assertEqual(nodes, expected, msg="Expected only measurements in index with " "no templates configured, got %s" % (nodes,))
def test_single_fetch_data(self): """Test single fetch data for a series by name""" node = list(self.finder.find_nodes(Query(self.series1)))[0] time_info, data = node.reader.fetch(int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue(self.steps == len(data), msg="Expected %s datapoints, got %s instead" % ( self.steps, len(data),)) datapoints = [v for v in data if v] self.assertTrue(len(datapoints) == self.num_datapoints, msg="Expected %s datapoints - got %s" % ( self.num_datapoints, len(datapoints),))
def test_field_data_part_or_no_template_match(self): del self.finder measurements = ['test'] fields = {'field1': 1, 'field2': 2} tags = {'env': 'my_env', 'region': 'my_region', 'dc': 'dc1' } self.client.drop_database(self.db_name) self.client.create_database(self.db_name) self.write_data(measurements, tags, fields) self.config['influxdb']['templates'] = ['env.template_tag.measurement.field*'] self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('*') nodes = [n.name for n in self.finder.find_nodes(query)] expected = [] self.assertEqual(nodes, expected) self.config['influxdb']['templates'] = ['env.template_tag.measurement.field*', 'env.region.measurement.field*'] self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('*') nodes = sorted([n.path for n in self.finder.find_nodes(query)]) expected = [tags['env']] self.assertEqual(nodes, expected) query = Query('*.*') nodes = sorted([n.path for n in self.finder.find_nodes(query)]) expected = sorted(['.'.join([tags['env'], tags['region']])]) self.assertEqual(nodes, expected) query = Query('*.*.*') nodes = sorted([n.path for n in self.finder.find_nodes(query)]) expected = sorted(['.'.join([tags['env'], tags['region'], measurements[0]])]) self.assertEqual(nodes, expected) query = Query('*.*.*.*') nodes = sorted([n.path for n in self.finder.find_nodes(query)]) expected = sorted(['.'.join([tags['env'], tags['region'], measurements[0], f]) for f in fields.keys()]) self.assertEqual(nodes, expected)
def test_template_measurement_no_tags(self): template = "env.host.measurement.field*" del self.finder measurements = ['cpuusage'] fields = {'value': 1} tags = {'host': 'my_host1', 'env': 'my_env1', } metrics = ['.'.join([tags['env'], tags['host'], m]) for m in measurements] self.client.drop_database(self.db_name) self.client.create_database(self.db_name) self.write_data(measurements, tags, fields) fields = {'value': 1, } self.write_data(measurements, {}, fields) self.config['influxdb']['templates'] = [template] self.finder = influxgraph.InfluxDBFinder(self.config) nodes = [n.name for n in self.finder.find_nodes(Query('*'))] expected = [tags['env']] self.assertEqual(nodes, expected) cpu_nodes = list(self.finder.find_nodes(Query('my_env1.my_host1.*'))) expected = measurements self.assertEqual([n.name for n in cpu_nodes], expected)
def test_series_loader(self): query = Query('*') loader_memcache_key = influxgraph.utils.gen_memcache_pattern_key("_".join([ query.pattern, str(self.default_nodes_limit), str(0)])) del self.finder _loader_interval = 2 config = { 'influxdb' : { 'host' : 'localhost', 'port' : 8086, 'user' : 'root', 'pass' : 'root', 'db' : self.db_name, 'series_loader_interval': _loader_interval, 'memcache' : { 'host': 'localhost', 'ttl' : 60, 'max_value': 20, }, 'log_level': 0, 'fill': 'previous', },} try: _memcache = memcache.Client([config['influxdb']['memcache']['host']]) _memcache.delete(SERIES_LOADER_MUTEX_KEY) except NameError: pass finder = influxgraph.InfluxDBFinder(config) time.sleep(_loader_interval/2.0) # if finder.memcache: # self.assertTrue(finder.memcache.get(SERIES_LOADER_MUTEX_KEY)) self.assertTrue(finder.memcache) self.assertEqual(finder.memcache_ttl, 60, msg="Configured TTL of %s sec, got %s sec TTL instead" % ( 60, finder.memcache_ttl,)) self.assertEqual(finder.memcache.server_max_value_length, 1024**2*20, msg="Configured max value of %s MB, got %s instead" % ( 1024**2*20, finder.memcache.server_max_value_length,)) # Give series loader more than long enough to finish time.sleep(_loader_interval + 2) if finder.memcache: self.assertTrue(finder.memcache.get(loader_memcache_key)) del finder config['influxdb']['loader_startup_block'] = False finder = influxgraph.InfluxDBFinder(config) try: self.assertTrue(_SERIES_LOADER_LOCK.acquire(block=False)) except: pass else: _SERIES_LOADER_LOCK.release()
def test_templated_data_query(self): serie = self.graphite_series[0] nodes = list(self.finder.find_nodes(Query(serie))) time_info, data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue(serie in data, msg="Did not get data for requested series %s - got data for %s" % ( serie, data.keys(),)) self.assertEqual(time_info, (int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s")), self.step), msg="Time info and step do not match our requested values") datapoints = [v for v in data[serie] if v] self.assertTrue(len(datapoints) == self.num_datapoints, msg="Expected %s datapoints for %s - got %s" % ( self.num_datapoints, serie, len(datapoints),))
def test_multi_fetch_data(self): """Test fetching data for a single series by name""" nodes = list(self.finder.find_nodes(Query(self.series1))) time_info, data = self.finder.fetch_multi(nodes, int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s"))) self.assertTrue(self.series1 in data, msg="Did not get data for requested series %s - got data for %s" % ( self.series1, data.keys(),)) self.assertEqual(time_info, (int(self.start_time.strftime("%s")), int(self.end_time.strftime("%s")), self.step), msg="Time info and step do not match our requested values") datapoints = [v for v in data[self.series1] if v] self.assertTrue(len(datapoints) == self.num_datapoints, msg="Expected %s datapoints - got %s" % ( self.num_datapoints, len(datapoints),))
def test_template_multiple_tags(self): self.client.drop_database(self.db_name) self.client.create_database(self.db_name) template = "*.disk. host.measurement.path.fstype.field*" measurement = 'disk' tags = {'host': 'my_host', 'path': '/', 'fstype': 'ext4', } fields = {'free': 1, 'used': 1, } self.write_data([measurement], tags, fields) self.config['influxdb']['templates'] = [template] self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('%s.%s.%s.%s.*' % ( tags['host'], measurement, tags['path'], tags['fstype'])) nodes = list(self.finder.find_nodes(query)) self.assertEqual(sorted([n.name for n in nodes]), sorted(fields.keys()))
def test_non_greedy_field(self): measurements = ['cpu-0', 'cpu-1', 'cpu-2', 'cpu-3'] fields = {'load': 1, 'idle': 1, 'usage': 1, 'user': 1, } tags = {'host': 'my_host', 'env': 'my_env', } data = [{ "measurement": measurement, "tags": tags, "time": _time, "fields": fields, } for measurement in measurements for _time in [ (self.end_time - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ"), (self.end_time - datetime.timedelta(minutes=2)).strftime("%Y-%m-%dT%H:%M:%SZ"), ]] metrics = ['.'.join([tags['host'], m, f]) for f in fields.keys() for m in measurements] self.client.drop_database(self.db_name) self.client.create_database(self.db_name) self.assertTrue(self.client.write_points(data)) template = "host.measurement.field" self.config['influxdb']['templates'] = [template] self.finder = influxgraph.InfluxDBFinder(self.config) query = Query('%s.*.*' % (tags['host'],)) nodes = list(self.finder.find_nodes(query)) node_paths = sorted([n.path for n in nodes]) _metrics = ['.'.join([tags['host'], m, f]) for f in fields.keys() if not '.' in f for m in measurements ] expected = sorted(_metrics) self.assertEqual(node_paths, expected, msg="Expected nodes %s from template with non-greedy field - got %s" % ( expected, node_paths))