def main(host='localhost', port=8086):
"""Instantiate a connection to the InfluxDB."""
user = '******'
password = '******'
dbname = 'example'
dbuser = '******'
dbuser_password = '******'
query = 'select Float_value from cpu_load_short;'
query_where = 'select Int_value from cpu_load_short where host=$host;'
bind_params = {'host': 'server01'}
json_body = [
{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"Float_value": 0.64,
"Int_value": 3,
"String_value": "Text",
"Bool_value": True
}
}
]
client = InfluxDBClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
print("Switch user: "******"Write points: {0}".format(json_body))
client.write_points(json_body)
print("Querying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print("Querying data: " + query_where)
result = client.query(query_where, bind_params=bind_params)
print("Result: {0}".format(result))
print("Switch user: "******"Drop database: " + dbname)
client.drop_database(dbname)
class TestInfluxDBClient(unittest.TestCase):
"""Set up the TestInfluxDBClient object."""
def setUp(self):
"""Initialize an instance of TestInfluxDBClient object."""
# By default, raise exceptions on warnings
warnings.simplefilter('error', FutureWarning)
self.cli = InfluxDBClient('localhost', 8086, 'username', 'password')
self.dummy_points = [
{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00.123456Z",
"fields": {
"value": 0.64
}
}
]
self.dsn_string = 'influxdb://*****:*****@my.host.fr:1886/db'
def test_scheme(self):
"""Set up the test schema for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
self.assertEqual('http://host:8086', cli._baseurl)
cli = InfluxDBClient(
'host', 8086, 'username', 'password', 'database', ssl=True
)
self.assertEqual('https://host:8086', cli._baseurl)
cli = InfluxDBClient(
'host', 8086, 'username', 'password', 'database', ssl=True,
path="somepath"
)
self.assertEqual('https://host:8086/somepath', cli._baseurl)
cli = InfluxDBClient(
'host', 8086, 'username', 'password', 'database', ssl=True,
path=None
)
self.assertEqual('https://host:8086', cli._baseurl)
cli = InfluxDBClient(
'host', 8086, 'username', 'password', 'database', ssl=True,
path="/somepath"
)
self.assertEqual('https://host:8086/somepath', cli._baseurl)
def test_dsn(self):
"""Set up the test datasource name for TestInfluxDBClient object."""
cli = InfluxDBClient.from_dsn('influxdb://192.168.0.1:1886')
self.assertEqual('http://192.168.0.1:1886', cli._baseurl)
cli = InfluxDBClient.from_dsn(self.dsn_string)
self.assertEqual('http://my.host.fr:1886', cli._baseurl)
self.assertEqual('uSr', cli._username)
self.assertEqual('pWd', cli._password)
self.assertEqual('db', cli._database)
self.assertFalse(cli._use_udp)
cli = InfluxDBClient.from_dsn('udp+' + self.dsn_string)
self.assertTrue(cli._use_udp)
cli = InfluxDBClient.from_dsn('https+' + self.dsn_string)
self.assertEqual('https://my.host.fr:1886', cli._baseurl)
cli = InfluxDBClient.from_dsn('https+' + self.dsn_string,
**{'ssl': False})
self.assertEqual('http://my.host.fr:1886', cli._baseurl)
def test_switch_database(self):
"""Test switch database in TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
cli.switch_database('another_database')
self.assertEqual('another_database', cli._database)
def test_switch_user(self):
"""Test switch user in TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
cli.switch_user('another_username', 'another_password')
self.assertEqual('another_username', cli._username)
self.assertEqual('another_password', cli._password)
def test_write(self):
"""Test write in TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write(
{"database": "mydb",
"retentionPolicy": "mypolicy",
"points": [{"measurement": "cpu_load_short",
"tags": {"host": "server01",
"region": "us-west"},
"time": "2009-11-10T23:00:00Z",
"fields": {"value": 0.64}}]}
)
self.assertEqual(
m.last_request.body,
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000000000000\n",
)
def test_write_points(self):
"""Test write points for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points,
)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west '
'value=0.64 1257894000123456000\n',
m.last_request.body.decode('utf-8'),
)
def test_write_points_toplevel_attributes(self):
"""Test write points attrs for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points,
database='testdb',
tags={"tag": "hello"},
retention_policy="somepolicy"
)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west,tag=hello '
'value=0.64 1257894000123456000\n',
m.last_request.body.decode('utf-8'),
)
def test_write_points_batch(self):
"""Test write points batch for TestInfluxDBClient object."""
dummy_points = [
{"measurement": "cpu_usage", "tags": {"unit": "percent"},
"time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}},
{"measurement": "network", "tags": {"direction": "in"},
"time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}},
{"measurement": "network", "tags": {"direction": "out"},
"time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}}
]
expected_last_body = (
"network,direction=out,host=server01,region=us-west "
"value=12.0 1257894000000000000\n"
)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write_points(points=dummy_points,
database='db',
tags={"host": "server01",
"region": "us-west"},
batch_size=2)
self.assertEqual(m.call_count, 2)
self.assertEqual(expected_last_body,
m.last_request.body.decode('utf-8'))
def test_write_points_udp(self):
"""Test write points UDP for TestInfluxDBClient object."""
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient(
'localhost', 8086, 'root', 'root',
'test', use_udp=True, udp_port=port
)
cli.write_points(self.dummy_points)
received_data, addr = s.recvfrom(1024)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west '
'value=0.64 1257894000123456000\n',
received_data.decode()
)
@raises(Exception)
def test_write_points_fails(self):
"""Test write points fail for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points([])
def test_write_points_with_precision(self):
"""Test write points with precision for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points, time_precision='n')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456000\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='u')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='ms')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='s')
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000\n",
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='m')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 20964900\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='h')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 349415\n',
m.last_request.body,
)
def test_write_points_with_precision_udp(self):
"""Test write points with precision for TestInfluxDBClient object."""
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient(
'localhost', 8086, 'root', 'root',
'test', use_udp=True, udp_port=port
)
cli.write_points(self.dummy_points, time_precision='n')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456000\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='u')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='ms')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='s')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000\n",
received_data,
)
cli.write_points(self.dummy_points, time_precision='m')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 20964900\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='h')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 349415\n',
received_data,
)
def test_write_points_bad_precision(self):
"""Test write points w/bad precision TestInfluxDBClient object."""
cli = InfluxDBClient()
with self.assertRaisesRegexp(
Exception,
"Invalid time precision is given. "
"\(use 'n', 'u', 'ms', 's', 'm' or 'h'\)"
):
cli.write_points(
self.dummy_points,
time_precision='g'
)
@raises(Exception)
def test_write_points_with_precision_fails(self):
"""Test write points w/precision fail for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points_with_precision([])
def test_query(self):
"""Test query method for TestInfluxDBClient object."""
example_response = (
'{"results": [{"series": [{"measurement": "sdfsdfsdf", '
'"columns": ["time", "value"], "values": '
'[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": '
'[{"measurement": "cpu_load_short", "columns": ["time", "value"], '
'"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_query_chunked(self):
"""Test chunked query for TestInfluxDBClient object."""
cli = InfluxDBClient(database='db')
example_object = {
'points': [
[1415206250119, 40001, 667],
[1415206244555, 30001, 7],
[1415206228241, 20001, 788],
[1415206212980, 10001, 555],
[1415197271586, 10001, 23]
],
'measurement': 'foo',
'columns': [
'time',
'sequence_number',
'val'
]
}
example_response = \
json.dumps(example_object) + json.dumps(example_object)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/db/db/series",
text=example_response
)
self.assertListEqual(
cli.query('select * from foo', chunked=True),
[example_object, example_object]
)
@raises(Exception)
def test_query_fail(self):
"""Test query failed for TestInfluxDBClient object."""
with _mocked_session(self.cli, 'get', 401):
self.cli.query('select column_one from foo;')
def test_ping(self):
"""Test ping querying InfluxDB version."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/ping",
status_code=204,
headers={'X-Influxdb-Version': '1.2.3'}
)
version = self.cli.ping()
self.assertEqual(version, '1.2.3')
def test_create_database(self):
"""Test create database for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.create_database('new_db')
self.assertEqual(
m.last_request.qs['q'][0],
'create database "new_db"'
)
def test_create_numeric_named_database(self):
"""Test create db w/numeric name for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.create_database('123')
self.assertEqual(
m.last_request.qs['q'][0],
'create database "123"'
)
@raises(Exception)
def test_create_database_fails(self):
"""Test create database fail for TestInfluxDBClient object."""
with _mocked_session(self.cli, 'post', 401):
self.cli.create_database('new_db')
def test_drop_database(self):
"""Test drop database for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.drop_database('new_db')
self.assertEqual(
m.last_request.qs['q'][0],
'drop database "new_db"'
)
def test_drop_measurement(self):
"""Test drop measurement for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.drop_measurement('new_measurement')
self.assertEqual(
m.last_request.qs['q'][0],
'drop measurement "new_measurement"'
)
def test_drop_numeric_named_database(self):
"""Test drop numeric db for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.drop_database('123')
self.assertEqual(
m.last_request.qs['q'][0],
'drop database "123"'
)
def test_get_list_database(self):
"""Test get list of databases for TestInfluxDBClient object."""
data = {'results': [
{'series': [
{'name': 'databases',
'values': [
['new_db_1'],
['new_db_2']],
'columns': ['name']}]}
]}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(
self.cli.get_list_database(),
[{'name': 'new_db_1'}, {'name': 'new_db_2'}]
)
@raises(Exception)
def test_get_list_database_fails(self):
"""Test get list of dbs fail for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_database()
def test_get_list_measurements(self):
"""Test get list of measurements for TestInfluxDBClient object."""
data = {
"results": [{
"series": [
{"name": "measurements",
"columns": ["name"],
"values": [["cpu"], ["disk"]
]}]}
]
}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(
self.cli.get_list_measurements(),
[{'name': 'cpu'}, {'name': 'disk'}]
)
def test_create_retention_policy_default(self):
"""Test create default ret policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.create_retention_policy(
'somename', '1d', 4, default=True, database='db'
)
self.assertEqual(
m.last_request.qs['q'][0],
'create retention policy "somename" on '
'"db" duration 1d replication 4 shard duration 0s default'
)
def test_create_retention_policy(self):
"""Test create retention policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.create_retention_policy(
'somename', '1d', 4, database='db'
)
self.assertEqual(
m.last_request.qs['q'][0],
'create retention policy "somename" on '
'"db" duration 1d replication 4 shard duration 0s'
)
def test_alter_retention_policy(self):
"""Test alter retention policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
# Test alter duration
self.cli.alter_retention_policy('somename', 'db',
duration='4d')
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" duration 4d'
)
# Test alter replication
self.cli.alter_retention_policy('somename', 'db',
replication=4)
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" replication 4'
)
# Test alter shard duration
self.cli.alter_retention_policy('somename', 'db',
shard_duration='1h')
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" shard duration 1h'
)
# Test alter default
self.cli.alter_retention_policy('somename', 'db',
default=True)
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" default'
)
@raises(Exception)
def test_alter_retention_policy_invalid(self):
"""Test invalid alter ret policy for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.alter_retention_policy('somename', 'db')
def test_drop_retention_policy(self):
"""Test drop retention policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.drop_retention_policy('somename', 'db')
self.assertEqual(
m.last_request.qs['q'][0],
'drop retention policy "somename" on "db"'
)
@raises(Exception)
def test_drop_retention_policy_fails(self):
"""Test failed drop ret policy for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'delete', 401):
cli.drop_retention_policy('default', 'db')
def test_get_list_retention_policies(self):
"""Test get retention policies for TestInfluxDBClient object."""
example_response = \
'{"results": [{"series": [{"values": [["fsfdsdf", "24h0m0s", 2]],'\
' "columns": ["name", "duration", "replicaN"]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://*****:*****@mock.patch('requests.Session.request')
def test_request_retry(self, mock_request):
"""Test that two connection errors will be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self):
self.i = 0
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < 3:
raise requests.exceptions.ConnectionError
r = requests.Response()
r.status_code = 204
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points
)
@mock.patch('requests.Session.request')
def test_request_retry_raises(self, mock_request):
"""Test that three requests errors will not be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self):
self.i = 0
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < 4:
raise requests.exceptions.HTTPError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
with self.assertRaises(requests.exceptions.HTTPError):
cli.write_points(self.dummy_points)
@mock.patch('requests.Session.request')
def test_random_request_retry(self, mock_request):
"""Test that a random number of connection errors will be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self, retries):
self.i = 0
self.retries = retries
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < self.retries:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 204
return r
retries = random.randint(1, 5)
mock_request.side_effect = CustomMock(retries).connection_error
cli = InfluxDBClient(database='db', retries=retries)
cli.write_points(self.dummy_points)
@mock.patch('requests.Session.request')
def test_random_request_retry_raises(self, mock_request):
"""Test a random number of conn errors plus one will not be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self, retries):
self.i = 0
self.retries = retries
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < self.retries + 1:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
retries = random.randint(1, 5)
mock_request.side_effect = CustomMock(retries).connection_error
cli = InfluxDBClient(database='db', retries=retries)
with self.assertRaises(requests.exceptions.ConnectionError):
cli.write_points(self.dummy_points)
def test_get_list_users(self):
"""Test get users for TestInfluxDBClient object."""
example_response = (
'{"results":[{"series":[{"columns":["user","admin"],'
'"values":[["test",false]]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.assertListEqual(
self.cli.get_list_users(),
[{'user': '******', 'admin': False}]
)
def test_get_list_users_empty(self):
"""Test get empty userlist for TestInfluxDBClient object."""
example_response = (
'{"results":[{"series":[{"columns":["user","admin"]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.assertListEqual(self.cli.get_list_users(), [])
def test_grant_admin_privileges(self):
"""Test grant admin privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.grant_admin_privileges('test')
self.assertEqual(
m.last_request.qs['q'][0],
'grant all privileges to "test"'
)
@raises(Exception)
def test_grant_admin_privileges_invalid(self):
"""Test grant invalid admin privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.grant_admin_privileges('')
def test_revoke_admin_privileges(self):
"""Test revoke admin privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.revoke_admin_privileges('test')
self.assertEqual(
m.last_request.qs['q'][0],
'revoke all privileges from "test"'
)
@raises(Exception)
def test_revoke_admin_privileges_invalid(self):
"""Test revoke invalid admin privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.revoke_admin_privileges('')
def test_grant_privilege(self):
"""Test grant privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.grant_privilege('read', 'testdb', 'test')
self.assertEqual(
m.last_request.qs['q'][0],
'grant read on "testdb" to "test"'
)
@raises(Exception)
def test_grant_privilege_invalid(self):
"""Test grant invalid privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.grant_privilege('', 'testdb', 'test')
def test_revoke_privilege(self):
"""Test revoke privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/query",
text=example_response
)
self.cli.revoke_privilege('read', 'testdb', 'test')
self.assertEqual(
m.last_request.qs['q'][0],
'revoke read on "testdb" from "test"'
)
@raises(Exception)
def test_revoke_privilege_invalid(self):
"""Test revoke invalid privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.revoke_privilege('', 'testdb', 'test')
def test_get_list_privileges(self):
"""Tst get list of privs for TestInfluxDBClient object."""
data = {'results': [
{'series': [
{'columns': ['database', 'privilege'],
'values': [
['db1', 'READ'],
['db2', 'ALL PRIVILEGES'],
['db3', 'NO PRIVILEGES']]}
]}
]}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(
self.cli.get_list_privileges('test'),
[{'database': 'db1', 'privilege': 'READ'},
{'database': 'db2', 'privilege': 'ALL PRIVILEGES'},
{'database': 'db3', 'privilege': 'NO PRIVILEGES'}]
)
@raises(Exception)
def test_get_list_privileges_fails(self):
"""Test failed get list of privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_privileges('test')
def test_invalid_port_fails(self):
"""Test invalid port fail for TestInfluxDBClient object."""
with self.assertRaises(ValueError):
InfluxDBClient('host', '80/redir', 'username', 'password')
def test_chunked_response(self):
"""Test chunked reponse for TestInfluxDBClient object."""
example_response = \
u'{"results":[{"statement_id":0,"series":' \
'[{"name":"cpu","columns":["fieldKey","fieldType"],"values":' \
'[["value","integer"]]}],"partial":true}]}\n{"results":' \
'[{"statement_id":0,"series":[{"name":"iops","columns":' \
'["fieldKey","fieldType"],"values":[["value","integer"]]}],' \
'"partial":true}]}\n{"results":[{"statement_id":0,"series":' \
'[{"name":"load","columns":["fieldKey","fieldType"],"values":' \
'[["value","integer"]]}],"partial":true}]}\n{"results":' \
'[{"statement_id":0,"series":[{"name":"memory","columns":' \
'["fieldKey","fieldType"],"values":[["value","integer"]]}]}]}\n'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
response = self.cli.query('show series limit 4 offset 0',
chunked=True, chunk_size=4)
self.assertTrue(len(response) == 4)
self.assertEqual(response.__repr__(), ResultSet(
{'series': [{'values': [['value', 'integer']],
'name': 'cpu',
'columns': ['fieldKey', 'fieldType']},
{'values': [['value', 'integer']],
'name': 'iops',
'columns': ['fieldKey', 'fieldType']},
{'values': [['value', 'integer']],
'name': 'load',
'columns': ['fieldKey', 'fieldType']},
{'values': [['value', 'integer']],
'name': 'memory',
'columns': ['fieldKey', 'fieldType']}]}
).__repr__())
def main(file,
hostname='localhost',
port=8086,
username='******',
password='******',
database='freifunk'):
jsondata = {}
jsondata = read_jsonfile(file)
series = create_series(jsondata)
client = InfluxDBClient(hostname, port, username, password, database)
print("Create database: " + database)
try:
client.create_database(database)
except InfluxDBClientError:
print("Database already existing, skipping creation")
pass
print("Create a retention policy")
try:
retention_policy = 'freifunk_policy'
client.create_retention_policy(retention_policy, '3d', 1, default=True)
except InfluxDBClientError:
print("Retention policy existing, skipping creation")
pass
client.write_points(series, retention_policy=retention_policy)
print("Data written to influxdb!")
def start():
"""Opens connections with logger, InfluxDB and MQTT broker."""
global logger
global mqtt_client
global house_data
global config
global influx_client
logger = LoggerClient.open("InfluxDBHub")
mqtt_client = Utils.getpahoclient(logger, __configure_mqtt)
config = Utils.getconfig("influxdb", logger)
influx_client = InfluxDBClient(config["host"], config["port"],
config["user"], config["password"],
config["database"])
if not {"name": config["database"]} in influx_client.get_list_database():
influx_client.create_database(config["database"])
if not any([
x["name"] == "raspimon_policy"
for x in influx_client.get_list_retention_policies()
]):
influx_client.create_retention_policy('raspimon_policy',
config["retention_policy"],
1,
default=True)
else:
influx_client.alter_retention_policy(
'raspimon_policy',
duration=config["retention_policy"],
replication=1,
default=True)
class InfluxDBPublisher(yaml.YAMLObject, Publisher, metaclass=PublisherMeta):
""" InfluxDB Publisher
Publishing service that sends data to an InfluxDB instance.
"""
yaml_tag = u'!InfluxDBPublisher'
def __init__(self, host, port, database, username, password):
self.host = host
self.port = port
self.database = database
self.username = username
self.password = password
def activate(self):
self.influxdb_client = InfluxDBClient(self.host, self.port, self.username, self.password, None)
databases = self.influxdb_client.get_list_database()
if len(list(filter(lambda x: x['name'] == self.database, databases))) == 0:
self.influxdb_client.create_database(self.database)
self.influxdb_client.create_retention_policy('oneweek', '1w', 1, self.database, default=True)
self.influxdb_client.switch_database(self.database)
def publish(self, data: list):
for d in data:
self.influxdb_client.write_points([
{
'measurement': d.name,
'tags': {
'tag': d.tag,
},
'fields': {
'value': d.value
}
}
])
def main(host='localhost',
port=8086,
database='',
user='',
password='',
rtSource='autogen',
retentionPolicy='history',
groupTime='1h',
duration='INF'):
""" Instantiate a connection to the InfluxDB. """
client = InfluxDBClient(host, port, user, password, database, timeout=None)
if not list(
client.query("show retention policies").get_points(
tags={"name": retentionPolicy})):
client.create_retention_policy(retentionPolicy,
duration,
1,
database,
default=False,
shard_duration=u'1d')
print("Retention policy " + retentionPolicy + " created")
measurements = getMeasurements(client)
for measurement in measurements:
if not checkCq(client, database, retentionPolicy, measurement):
print("--> " + measurement)
fields = getFields(client, measurement)
nfields = countFields(fields.get_points())
line = ""
for field in fields.get_points():
if field["fieldType"] == "string":
line = line + "first(\"" + field[
"fieldKey"] + "\") as \"" + field["fieldKey"] + "\","
else:
if nfields <= 15:
line = line + "mean(\"" + field[
"fieldKey"] + "\") as \"" + field[
"fieldKey"] + "\","
else:
line = line + "mean(\"" + field[
"fieldKey"] + "\") as \"" + field[
"fieldKey"] + "\","
line = line + "max(\"" + field[
"fieldKey"] + "\") as \"" + field[
"fieldKey"] + "_max\","
CQNAME = "\"" + database + "_" + retentionPolicy + "_" + measurement + "\""
FROM = "\"" + database + "\".\"" + rtSource + "\".\"" + measurement + "\""
INTO = "\"" + database + "\".\"" + retentionPolicy + "\".\"" + measurement + "\""
Query = "SELECT " + line[:
-1] + " INTO " + INTO + " FROM " + FROM + " WHERE time < now()-" + groupTime + " GROUP BY time(" + groupTime + "),*"
QueryCQ = "CREATE CONTINUOUS QUERY " + CQNAME + " ON " + database + " BEGIN " + Query + " END "
print("\tCharging data on " + INTO)
client.query(Query)
print("\tCreating CQ: " + CQNAME)
client.query(QueryCQ)
class DBManager(object):
def __init__(self, server):
self.server = server
self.influx = InfluxDBClient(host=self.server.url,
port=self.server.port,
username=self.server.username,
password=self.server.password,
ssl=self.server.ssl,
database='varken',
verify_ssl=self.server.verify_ssl)
version = self.influx.request(
'ping', expected_response_code=204).headers['X-Influxdb-Version']
databases = [db['name'] for db in self.influx.get_list_database()]
self.logger = getLogger()
self.logger.info('Influxdb version: %s', version)
if 'varken' not in databases:
self.logger.info("Creating varken database")
self.influx.create_database('varken')
self.logger.info("Creating varken retention policy (30d/1h)")
self.influx.create_retention_policy('varken 30d/1h', '30d', '1',
'varken', False, '1h')
def write_points(self, data):
d = data
self.logger.debug('Writing Data to InfluxDB %s', d)
try:
self.influx.write_points(d)
except (InfluxDBServerError, ConnectionError) as e:
self.logger.error(
'Error writing data to influxdb. Dropping this set of data. '
'Check your database! Error: %s', e)
def main():
parser = argparse.ArgumentParser(description='DB setup')
parser.add_argument('--user', help='Username for DB')
parser.add_argument('--password', help='Password for DB')
parser.add_argument('--host', help='Host for connecting to DB')
parser.add_argument('--dbname', help='Database to insert data to')
parser.add_argument('--port', type=int, help='Port for connecting to DB')
parser.add_argument('--policyname', help='Policy for DB')
args = parser.parse_args()
user = args.user
password = args.password
host = args.host
port = args.port
db_name = args.dbname
policy_name = args.policyname
client = InfluxDBClient(host=host,
username=user,
password=password,
port=port)
try:
client.create_database(db_name)
client.create_retention_policy(policy_name,
'1d',
database=db_name,
replication='1',
default=False)
except Exception as exception:
logger.error(exception)
sys.exit(1)
logger.info(client.get_list_database())
def main(host='localhost', port=8086):
user = '******'
password = '******'
dbname = 'veseventsdb'
dbuser = '******'
dbuser_password = '******'
query = 'select value from cpu_load_short;'
json_body = [{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"Float_value": 0.64,
"Int_value": 3,
"String_value": "Text",
"Bool_value": True
}
}]
client = InfluxDBClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '6h', 3, default=True)
def create_db(self,host, port, user, password, dbname):
client = InfluxDBClient(host, port, user, password, dbname)
logger.debug("self ="+str(self))
try:
client.drop_database(dbname)
except Exception as e:
print("didnt drop old db")
logger.error("Didn't drop old db:"+str(e))
return False
try:
client.create_database(dbname)
except Exception as e:
print("db not created - {0}".format(e))
return False
except:
print("unknown error openning db")
return False
print("db created")
try:
client.create_retention_policy('infinite retention', 'INF', 3, default=True)
except Exception as e:
print("retention policy not set - {0}".format(e))
logger.error("retention policy not set - {0}".format(e))
return False
except:
print("unknown error openning db")
return False
client.close()
return True
def setup_db_for_use(host,
port,
db_name='firefly',
retention_duration='1h',
retention_policy_name="default_firefly_retention"):
"""
Sets up an instance of InfluxDB to store data to.
:param host: the ip which can be used to access the db
:param port: the port by which the db can be accessed
:param db_name: name of the databse to use inside InfluxDB
:param retention_duration: specifies how long to retain the stored information
:param retention_policy_name: name of the retention policy that is to be created
:return: an active InfluxDB client
"""
client = InfluxDBClient(host, port)
if {"name": db_name} not in client.get_list_database():
client.create_database(db_name)
client.switch_database(db_name)
client.create_retention_policy(retention_policy_name,
retention_duration,
1,
default=True)
return client
def prepare(host, port):
client = InfluxDBClient(host, port, user, password, dbname, pool_size=10)
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
def main():
try:
client = InfluxDBClient(HOST, 8086, USER, PWD, DBNAME)
client.create_database(DBNAME)
print("Create retention policy")
client.create_retention_policy('one_day_only', '1d', 1, default=True)
def send_json(database, json_data, policy_name, policy_duration):
client = InfluxDBClient(host='xxx')
client.create_retention_policy(policy_name, policy_duration, '2', database)
client.switch_database(database)
try:
client.write_points(json_data)
except Exception as e:
print(e)
def username(vmanage_session):
response = vmanage_session.get_request("admin/user")
items = response.json()['data']
#login credentials for InfluxDB
host = 'localhost'
port = 8086
USER = '******'
PASSWORD = '******'
DBNAME = 'username'
series = []
total_records = 0
json_body = {}
#loop over the API response variable items and create records to be stored in InfluxDB
for i in items:
#print('items')
#print(i)
if len(i) == 3:
json_body = {
"measurement": "usernames",
"tags": {
"username": str(i['userName']),
},
#"time": time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime(i['entry_time']/1000.)),
"fields": {
"fullname": str(i['description']),
"group": str(i['group'])
}
}
series.append(json_body)
total_records = total_records + 1
else:
json_body = {
"measurement": "usernames",
"tags": {
"username": str(i['userName']),
},
#"time": time.strftime('%m/%d/%Y %H:%M:%S', time.gmtime(i['entry_time']/1000.)),
"fields": {
"group": str(i['group'])
}
}
series.append(json_body)
total_records = total_records + 1
client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)
print("Create a retention policy")
retention_policy = 'retention_policy_1'
client.create_retention_policy(retention_policy, '10d', 3, default=True)
print("Write points #: {0}".format(total_records))
client.write_points(series, retention_policy=retention_policy)
time.sleep(2)
def main(host='localhost', port=8086, nb_day=15):
"""Instantiate a connection to the backend."""
nb_day = 15 # number of day to generate time series
timeinterval_min = 5 # create an event every x minutes
total_minutes = 1440 * nb_day
total_records = int(total_minutes / timeinterval_min)
now = datetime.datetime.today()
metric = "server_data.cpu_idle"
series = []
for i in range(0, total_records):
past_date = now - datetime.timedelta(minutes=i * timeinterval_min)
value = random.randint(0, 200)
hostName = "server-%d" % random.randint(1, 5)
# pointValues = [int(past_date.strftime('%s')), value, hostName]
pointValues = {
"time": int(past_date.strftime('%s')),
"measurement": metric,
"fields": {
"value": value,
},
"tags": {
"hostName": hostName,
},
}
series.append(pointValues)
print(series)
client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)
print("Create database: " + DBNAME)
try:
client.create_database(DBNAME)
except InfluxDBClientError:
# Drop and create
client.drop_database(DBNAME)
client.create_database(DBNAME)
print("Create a retention policy")
retention_policy = 'server_data'
client.create_retention_policy(retention_policy, '3d', 3, default=True)
print("Write points #: {0}".format(total_records))
client.write_points(series, retention_policy=retention_policy)
time.sleep(2)
query = "SELECT MEAN(value) FROM {} WHERE \
time > now() - 10d GROUP BY time(500m)".format(metric)
result = client.query(query, database=DBNAME)
print(result)
print("Result: {0}".format(result))
print("Drop database: {}".format(DBNAME))
client.drop_database(DBNAME)
class DBManager(object):
def __init__(self, server):
self.server = server
self.logger = getLogger()
if self.server.url == "influxdb.domain.tld":
self.logger.critical(
"You have not configured your varken.ini. Please read Wiki page for configuration"
)
exit()
self.influx = InfluxDBClient(host=self.server.url,
port=self.server.port,
username=self.server.username,
password=self.server.password,
ssl=self.server.ssl,
database='varken',
verify_ssl=self.server.verify_ssl)
try:
version = self.influx.request(
'ping',
expected_response_code=204).headers['X-Influxdb-Version']
self.logger.info('Influxdb version: %s', version)
except ConnectionError:
self.logger.critical(
"Error testing connection to InfluxDB. Please check your url/hostname"
)
exit()
databases = [db['name'] for db in self.influx.get_list_database()]
if 'varken' not in databases:
self.logger.info("Creating varken database")
self.influx.create_database('varken')
retention_policies = [
policy['name']
for policy in self.influx.get_list_retention_policies(
database='varken')
]
if 'varken 30d-1h' not in retention_policies:
self.logger.info("Creating varken retention policy (30d-1h)")
self.influx.create_retention_policy(name='varken 30d-1h',
duration='30d',
replication='1',
database='varken',
default=True,
shard_duration='1h')
def write_points(self, data):
d = data
self.logger.debug('Writing Data to InfluxDB %s', d)
try:
self.influx.write_points(d)
except (InfluxDBServerError, ConnectionError) as e:
self.logger.error(
'Error writing data to influxdb. Dropping this set of data. '
'Check your database! Error: %s', e)
def send_stats_to_influxdb(parsed_stats, timestamp):
try:
db_client = InfluxDBClient(influx_ip, influx_port, influx_username, influx_password, influx_database)
if not {u'name': u'dslstats'} in db_client.get_list_database():
db_client.create_database("dslstats")
db_client.create_retention_policy("dslstats-retention-policy", "52w", "1", default=True)
json = format_json(parsed_stats, timestamp)
db_client.write_points(json)
except Exception:
raise
def network_detect_task_query(request):
""" 查询探测任务 """
hostname = request.POST.get('hostname', '')
sn = request.POST.get('sn', '')
if not sn or not hostname:
response_data = {'success': False, 'msg': u"未传递主机名和sn号"}
return HttpResponse(json.dumps(response_data),
content_type="application/json")
client = InfluxDBClient(IP_INFLUXDB, PORT_INFLUXDB, USER_INFLUXDB,
PASSWORD_INFLUXDB)
string_db_name = 'network_detect_task'
client.create_database(string_db_name)
string_db_retention_policy = 'auto_delte_1d'
client.create_retention_policy(string_db_retention_policy,
database=string_db_name,
duration='1d',
replication=REPLICATION_INFLUXDB,
default=True)
sql = """select * from %s where time>now() - %s and hostname='%s' and sn='%s' limit 1;""" % (
string_db_name, EXPIRE_NETWORK_DETECT_TASK, hostname, sn)
sql_result = client.query(sql, database=string_db_name)
list_task = list(sql_result.get_points())
if list_task:
response_data = list_task[0]['data']
if isinstance(response_data, str):
response_data = json.loads(response_data)
return HttpResponse(json.dumps(response_data),
content_type="application/json")
else:
data = {"hostname": hostname, "sn": sn}
response_data = do_request(URL_NETWORK_DETECT_TASK, data=data)
if response_data['success']:
localtime = time.strftime(
'%Y-%m-%dT%H:%M:00Z',
time.localtime(float(time.time()) - 8 * 60 * 60))
sql_json = [{
"measurement": string_db_name,
"tags": {
"hostname": hostname,
"sn": sn,
},
"time": localtime,
"fields": {
"data": json.dumps(response_data)
}
}]
client.write_points(sql_json,
database=string_db_name,
retention_policy=string_db_retention_policy)
return HttpResponse(json.dumps(response_data),
content_type="application/json")
else:
response_data = {'success': False, 'msg': u"获取探测任务失败"}
return HttpResponse(json.dumps(response_data),
content_type="application/json")
def main(host='localhost', port=8086, nb_day=15):
nb_day = 15 # number of day to generate time series
timeinterval_min = 5 # create an event every x minutes
total_minutes = 1440 * nb_day
total_records = int(total_minutes / timeinterval_min)
now = datetime.datetime.today()
metric = "server_data.cpu_idle"
series = []
for i in range(0, total_records):
past_date = now - datetime.timedelta(minutes=i * timeinterval_min)
value = random.randint(0, 200)
hostName = "server-%d" % random.randint(1, 5)
# pointValues = [int(past_date.strftime('%s')), value, hostName]
pointValues = {
"time": int(past_date.strftime('%s')),
"measurement": metric,
'fields': {
'value': value,
},
'tags': {
"hostName": hostName,
},
}
series.append(pointValues)
print(series)
client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)
print("Create database: " + DBNAME)
try:
client.create_database(DBNAME)
except InfluxDBClientError:
# Drop and create
client.drop_database(DBNAME)
client.create_database(DBNAME)
print("Create a retention policy")
retention_policy = 'server_data'
client.create_retention_policy(retention_policy, '3d', 3, default=True)
print("Write points #: {0}".format(total_records))
client.write_points(series, retention_policy=retention_policy)
time.sleep(2)
query = "SELECT MEAN(value) FROM {} WHERE time > now() - 10d GROUP BY time(500m)".format(metric)
result = client.query(query, database=DBNAME)
print(result)
print("Result: {0}".format(result))
print("Drop database: {}".format(DBNAME))
client.drop_database(DBNAME)
class InfluxDB(DBAdaptor):
def __init__(self):
conf = cper.read_yaml_file('influxdb')
host = conf['host']
port = conf['port']
username = conf['username']
pwd = conf['password']
self.database = conf['database']
use_udp = conf['use_udp']
udp_port = conf['udp_port']
self.client = InfluxDBClient(host=host, port=port, username=username, password=pwd, database=self.database)
if self.client is None:
print('ERROR! No InfluxDB connection was built!')
else:
self.__setting()
def saveall(self, data):
rsl = self.client.write_points(data)
if not rsl:
print('Fail to saveall!')
def exec_sql(self, sql):
rsl = self.client.query(sql)
points = rsl.get_points()
return points
def __setting(self):
"""
1\ create database if not existed
2\ create rp if not existed
:return:
"""
rsl = self.client.get_list_database()
if {'name': self.database} not in rsl:
self.client.create_database(dbname=self.database)
rsl = self.client.get_list_retention_policies(database=self.database)
is_rp_7d_existed = False
is_rp_30d_existed = False
for i in rsl:
if i['name'] == 'rp_7d':
is_rp_7d_existed = True
elif i['name'] == 'rp_30d':
is_rp_30d_existed = True
else:
continue
if not is_rp_7d_existed:
self.client.create_retention_policy(name='rp_7d', duration='168h', replication='1', database=self.database,
default=True, shard_duration='24h')
if not is_rp_30d_existed:
self.client.create_retention_policy(name='rp_30d', duration='720h', replication='1', database=self.database,
default=False, shard_duration='24h')
def __del__(self):
pass
def init_influxdb(self):
client = InfluxDBClient(
host=self.influxdb_host,
port=self.influxdb_port,
username=self.influxdb_user,
password=self.influxdb_user_pass,
database=self.influxdb_database,
)
try:
logging.debug('Testing InfluxDB connection')
version = client.request(
'ping',
expected_response_code=204).headers['X-Influxdb-Version']
logging.debug(f'Influxdb version: {version}')
except ConnectionError as e:
logging.critical(
f'Error testing connection to InfluxDB. Please check your url/hostname.\nError: {e}'
)
raise
try:
databases = [db['name'] for db in client.get_list_database()]
if self.influxdb_database in databases:
logging.debug(f'Found database: {self.influxdb_database}')
except InfluxDBClientError as e:
logging.critical(
f'Error getting database list! Please check your InfluxDB configuration.\nError: {e}'
)
raise
if self.influxdb_database not in databases:
logging.info(f'Creating database: {self.influxdb_database}')
client.create_database(self.influxdb_database)
retention_policies = [
policy['name']
for policy in client.get_list_retention_policies(
database=self.influxdb_database)
]
if f'{self.influxdb_database} {self.influxdb_retention}-{self.influxdb_shard}' not in retention_policies:
logging.info(
f'Creating {self.influxdb_database} retention policy ({self.influxdb_retention}-{self.influxdb_shard})'
)
client.create_retention_policy(
name=
f'{self.influxdb_database} {self.influxdb_retention}-{self.influxdb_shard}',
duration=self.influxdb_retention,
replication='1',
database=self.influxdb_database,
default=True,
shard_duration=self.influxdb_shard)
return client
class dbWrapper():
def __init__(self, host, port, user, password, dbname, dbuser, dbpss):
self.port = port
self.host = host
self.user = user
self.password = password
self.dbname = dbname
self.dbuser = dbuser
self.dbpss = dbpss
self.tryReconnect()
def tryReconnect(self):
helper.internalLogger.info("Try reconnection to database" +
self.dbname)
try:
"""Instantiate a connection to the InfluxDB."""
self.client = InfluxDBClient(self.host, self.port, self.user,
self.password, self.dbname)
helper.internalLogger.info("Create database: " + self.dbname)
self.client.create_database(self.dbname)
helper.internalLogger.info("Create a retention policy")
self.client.create_retention_policy('awesome_policy',
'3d',
3,
default=True)
helper.internalLogger.info("Switch user: "******"Ok ok, quitting")
sys.exit(1)
except Exception as e:
e = sys.exc_info()[0]
helper.internalLogger.error(
'Unexpected error attempting to access to BD. It will be retried later.'
)
helper.einternalLogger.exception(e)
def addData(self, data):
try:
json_body = data
helper.internalLogger.debug("Adding to DB: {0}".format(data))
self.client.write_points(json_body)
except KeyboardInterrupt:
print("Ok ok, quitting")
sys.exit(1)
except Exception as e:
e = sys.exc_info()[0]
helper.internalLogger.error('Unexpected error inserting in DB')
helper.einternalLogger.exception(e)
self.tryReconnect()
def loop(ds18b20):
#mail
smtp_server = "smtp.gmail.com"
portsmtp = 465
sender_email = "*****@*****.**"
receiver_email = ["*****@*****.**"]
password = "******"
#database
USER = '******'
PASSWORD = '******'
DBNAME = 'temp'
host='localhost'
hostName = 'pi'
port=8086
metric = "Server Room Temperatures"
retention_policy = 'awesome_policy'
grafana_timezone = pytz.utc
while True:
client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)
client.create_retention_policy(retention_policy, '30d', 3, default=True)
json_temp = {
"tags": {
"hostName": hostName
},
"measurement": metric,
"fields": {
"value": read(ds18b20)[0]
},
"time": datetime.datetime.now(grafana_timezone).strftime("%m/%d/%Y, %H:%M:%S"),
}
#display
with canvas(device) as draw:
draw.text((0, 0), str(json_temp['fields']['value']), fill="white",font=font)
with open ('last_breath','w') as file:
file.write (str(json_temp))
client.write_points([json_temp], retention_policy=retention_policy)
if read(ds18b20) != None and read(ds18b20)[0] > 35:
message = "Temperatura w serwerowni wynosi {}".format(read(ds18b20)[0])
with smtplib.SMTP_SSL(smtp_server, portsmtp) as server:
server.login(sender_email, password)
server.sendmail(sender_email, receiver_email, message)
server.close()
time.sleep(180)
def send_to_db(json_body, host='localhost', port=8086):
"""Instantiate a connection to the InfluxDB."""
user = '******'
password = '******'
dbname = 'letna'
client = InfluxDBClient(host, port, user, password, dbname)
#print("Create database: " + dbname)
#client.create_database(dbname)
#print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
#print("Write points: {0}".format(json_body))
response = client.write_points(json_body)
def init_influx():
""" Initializes influx database. """
print("Connecting to Influx...")
influx_client = InfluxDBClient(
host=config.INFLUX_HOST,
port=config.INFLUX_PORT,
database=config.INFLUX_DB_NAME,
)
print("Connected to Influx!")
print("Creating influx database...")
influx_client.create_database(config.INFLUX_DB_NAME)
influx_client.create_retention_policy("one_week", "1w", 1, "listenbrainz")
print("Done!")
def main(host='localhost', port=8086):
user = '******'
password = '******'
dbname = 'example'
dbuser = '******'
dbuser_password = '******'
query = 'select value from cpu_load_short;'
json_body = [
{
"name": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"timestamp": "2009-11-10T23:00:00Z",
"fields": {
"value": 0.64
}
}
]
client = InfluxDBClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
print("Switch user: "******"Write points: {0}".format(json_body))
client.write_points(json_body)
print("Queying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print("Switch user: "******"Drop database: " + dbname)
client.drop_database(dbname)
def main(host='localhost', port=8086):
user = '******'
password = '******'
dbname = 'example'
dbuser = '******'
dbuser_password = '******'
query = 'select * from test' #table
client = InfluxDBClient(host, port, user, password, dbname)
#Create
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
print("Switch user: "******"Update\n"
#Update
loop = 0
while loop < 1000:
loop = loop + 1
time.sleep(2);
print("Write points: {0}".format(get_Json()))
client.write_points(get_Json())
print("\nQueying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print "\nRetrieve\n"
#Retrieve
result = client.query('SELECT * FROM test WHERE BoolVal=true')
print("Result: {0}".format(result))
print("Switch user: "******"Drop database: " + dbname)
def main(file, hostname='localhost', port=8086, username='******', password='******', database='freifunk'):
jsondata = {}
jsondata=read_jsonfile(file)
series=create_series(jsondata)
client = InfluxDBClient(hostname, port, username, password, database)
print("Create database: " + database)
try:
client.create_database(database)
except InfluxDBClientError:
print("Database already existing, skipping creation")
pass
print("Create a retention policy")
try:
retention_policy = 'freifunk_policy'
client.create_retention_policy(retention_policy, '3d', 1, default=True)
except InfluxDBClientError:
print("Retention policy existing, skipping creation")
pass
client.write_points(series, retention_policy=retention_policy)
print("Data written to influxdb!")
def start():
"""Opens connections with logger, InfluxDB and MQTT broker."""
global logger
global mqtt_client
global house_data
global config
global influx_client
logger = LoggerClient.open("InfluxDBHub")
mqtt_client = Utils.getpahoclient(logger, __configure_mqtt)
config = Utils.getconfig("influxdb", logger)
influx_client = InfluxDBClient(config["host"], config["port"],
config["user"], config["password"],
config["database"])
if not {"name":config["database"]} in influx_client.get_list_database():
influx_client.create_database(config["database"])
if not any([ x["name"]=="raspimon_policy" for x in influx_client.get_list_retention_policies()]):
influx_client.create_retention_policy('raspimon_policy',
config["retention_policy"],
1, default=True)
else:
influx_client.alter_retention_policy('raspimon_policy',
duration=config["retention_policy"],
replication=1,
default=True)
def init_db():
global db
db = InfluxDBClient('localhost', 8086, 'root', 'root')
db.create_database('alan',if_not_exists=True)
db.switch_database('alan')
db.create_retention_policy('raw_hits', '1h', 1, database='alan', default=True)
db.create_retention_policy('workingset', '1h', 1, database='alan', default=False)
db.create_retention_policy('digest', '1d', 1, database='alan', default=False)
db.create_retention_policy('trends', '1w', 1, database='alan', default=False)
db.create_retention_policy('historical', '52w', 1, database='alan', default=False)
# downsample data into counts per 5 for all articles
# keep for a 1h
db.query( "DROP CONTINUOUS QUERY pageview_5s_count ON alan" );
db.query( "CREATE CONTINUOUS QUERY pageview_5s_count ON alan BEGIN SELECT count(\"duration\") INTO alan.workingset.pageview_5s_bins FROM alan.raw_hits.pageview GROUP BY site, section, url, time(5s) END" );
# work out pages/sec for all content, updated per minute
# keep for a day
db.query( "DROP CONTINUOUS QUERY create_pageview_rate ON alan" );
db.query( "CREATE CONTINUOUS QUERY create_pageview_rate ON alan BEGIN SELECT sum(\"count\")/60 INTO alan.digest.pageview_rate FROM alan.workingset.pageview_5s_bins GROUP BY site, section, url, time(1m) END" );
return db
def main(host='localhost', port=8086, client_num=1):
print("influxDB test start-------------")
print "host = {}".format(host)
print "port = {}".format(port)
print("LINE PROTOCOL")
hostnames = []
for i in xrange(0, NUMBER_OF_HOSTS):
hostnames.append(uuid.uuid4().hex)
user = '******'
password = '******'
db_name = 'monasca'
client = InfluxDBClient(host, port, user, password, db_name)
running_recording = False
if db_name not in client.get_list_database():
print "Create database: ".format(db_name)
client.create_database(db_name)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
print "Start recording top output"
running_recording = True
top_process = subprocess.Popen("exec top -b -d 1 > " + './' + 'system_info', shell=True)
db_user = '******'
db_user_password = '******'
print "Switch user: {}".format(db_user)
client.switch_user(db_user, db_user_password)
# INSERT
print "Write points: batch_size = {0}".format(NUMBER_PER_BATCH)
start_time = datetime.utcnow()
print "Start time: {0}".format(start_time)
dimension_keys_values_map = {'service': 'monitoring', 'host': 'localhost',
'cloud': 'cloud_test'}
print "Inserting {0} measurements".format(NUMBER_OF_MEASUREMENTS)
metric_count = 0
for i in xrange(NUMBER_OF_MEASUREMENTS / NUMBER_PER_BATCH):
batch_set = []
for j in xrange(NUMBER_PER_BATCH):
metric_suffix = metric_count % NUMBER_OF_UNIQUE_METRICS
metric_name = 'metric_KS_{}_'.format(client_num) + str(metric_suffix)
value = i * NUMBER_PER_BATCH + j
dims = dict(dimension_keys_values_map)
host_name = hostnames[(metric_count / NUMBER_OF_UNIQUE_METRICS) % NUMBER_OF_HOSTS]
dims['host'] = host_name
dimension_hash_string = ','.join(['%s=%s' % (d, dims[d]) for d in dims])
new_hash_string = REGION + TENANT_ID + metric_name + dimension_hash_string
sha1_hash = hashlib.sha1(new_hash_string).hexdigest()
metric_id = str(sha1_hash)
line_body = '{0},zone=nova,service=compute,resource_id=34c0ce14-9ce4-4d3d-84a4-172e1ddb26c4,' \
'tenant_id=71fea2331bae4d98bb08df071169806d,hostname={1},component=vm,' \
'control_plane=ccp,cluster=compute,cloud_name=monasca value={2},' \
'metric_id="{3}"'.format(metric_name, host_name, value, str(metric_id))
batch_set.append(line_body)
metric_count += 1
client.write_points(batch_set, batch_size=NUMBER_PER_BATCH,
time_precision='ms', protocol='line')
end_time = datetime.utcnow()
elapsed = end_time - start_time
if running_recording:
os.kill(top_process.pid, 9)
# Calculate Insert Rate
print "elapsed time: {0}".format(str(elapsed))
print "measurements per sec: {0}".format(str(float(NUMBER_OF_MEASUREMENTS) / elapsed.seconds))
config = config_parser['DEFAULT']
myclient = InfluxDBClient(
config['influx_remote_host'],
int(config['influx_remote_port']),
config['influx_remote_user'],
config['influx_remote_password'],
'pm',
config['influx_remote_https'] == 'true',
True,
timeout=30
)
myclient.create_database('pm')
myclient.create_retention_policy('pm_policy', 'INF', 3, default=True)
myclient.create_retention_policy('event_policy', 'INF', 3, default=False)
class PMSeriesHelper(SeriesHelper):
class Meta:
client = myclient
series_name = 'particulates'
fields = ['pm_25', 'pm_10']
tags = ['sensor_id']
bulk_size = 1
autocommit = True
def store(data):
PMSeriesHelper(sensor_id=config['sensor_id'], pm_25=data[0], pm_10=data[1])
class Output(cowrie.core.output.Output):
def __init__(self):
cowrie.core.output.Output.__init__(self)
def start(self):
try:
host = CONFIG.get('output_influx', 'host')
except Exception:
host = ''
try:
port = CONFIG.getint('output_influx', 'port')
except Exception:
port = 8086
try:
ssl = CONFIG.getboolean('output_influx', 'ssl')
except Exception:
ssl = False
self.client = None
try:
self.client = InfluxDBClient(host=host, port=port, ssl=ssl, verify_ssl=ssl)
except InfluxDBClientError as e:
log.err("output_influx: I/O error({0}): '{1}'".format(
e.errno, e.strerror))
return
if self.client is None:
log.err("output_influx: cannot instantiate client!")
return
if (CONFIG.has_option('output_influx', 'username') and
CONFIG.has_option('output_influx', 'password')):
username = CONFIG.get('output_influx', 'username')
password = CONFIG.get('output_influx', 'password', raw=True)
self.client.switch_user(username, password)
try:
dbname = CONFIG.get('output_influx', 'database_name')
except Exception:
dbname = 'cowrie'
retention_policy_duration_default = '12w'
retention_policy_name = dbname + "_retention_policy"
if CONFIG.has_option('output_influx', 'retention_policy_duration'):
retention_policy_duration = CONFIG.get(
'output_influx', 'retention_policy_duration')
match = re.search(r'^\d+[dhmw]{1}$', retention_policy_duration)
if not match:
log.err(("output_influx: invalid retention policy."
"Using default '{}'..").format(
retention_policy_duration))
retention_policy_duration = retention_policy_duration_default
else:
retention_policy_duration = retention_policy_duration_default
database_list = self.client.get_list_database()
dblist = [str(elem['name']) for elem in database_list]
if dbname not in dblist:
self.client.create_database(dbname)
self.client.create_retention_policy(
retention_policy_name, retention_policy_duration, 1,
database=dbname, default=True)
else:
retention_policies_list = self.client.get_list_retention_policies(
database=dbname)
rplist = [str(elem['name']) for elem in retention_policies_list]
if retention_policy_name not in rplist:
self.client.create_retention_policy(
retention_policy_name, retention_policy_duration, 1,
database=dbname, default=True)
else:
self.client.alter_retention_policy(
retention_policy_name, database=dbname,
duration=retention_policy_duration,
replication=1, default=True)
self.client.switch_database(dbname)
def stop(self):
pass
def write(self, entry):
if self.client is None:
log.err("output_influx: client object is not instantiated")
return
# event id
eventid = entry['eventid']
# measurement init
m = {
'measurement': eventid.replace('.', '_'),
'tags': {
'session': entry['session'],
'src_ip': entry['src_ip']
},
'fields': {
'sensor': self.sensor
},
}
# event parsing
if eventid in ['cowrie.command.failed',
'cowrie.command.input']:
m['fields'].update({
'input': entry['input'],
})
elif eventid == 'cowrie.session.connect':
m['fields'].update({
'protocol': entry['protocol'],
'src_port': entry['src_port'],
'dst_port': entry['dst_port'],
'dst_ip': entry['dst_ip'],
})
elif eventid in ['cowrie.login.success', 'cowrie.login.failed']:
m['fields'].update({
'username': entry['username'],
'password': entry['password'],
})
elif eventid == 'cowrie.session.file_download':
m['fields'].update({
'shasum': entry.get('shasum'),
'url': entry.get('url'),
'outfile': entry.get('outfile')
})
elif eventid == 'cowrie.session.file_download.failed':
m['fields'].update({
'url': entry.get('url')
})
elif eventid == 'cowrie.session.file_upload':
m['fields'].update({
'shasum': entry.get('shasum'),
'outfile': entry.get('outfile'),
})
elif eventid == 'cowrie.session.closed':
m['fields'].update({
'duration': entry['duration']
})
elif eventid == 'cowrie.client.version':
m['fields'].update({
'version': ','.join(entry['version']),
})
elif eventid == 'cowrie.client.kex':
m['fields'].update({
'maccs': ','.join(entry['macCS']),
'kexalgs': ','.join(entry['kexAlgs']),
'keyalgs': ','.join(entry['keyAlgs']),
'compcs': ','.join(entry['compCS']),
'enccs': ','.join(entry['encCS'])
})
elif eventid == 'cowrie.client.size':
m['fields'].update({
'height': entry['height'],
'width': entry['width'],
})
elif eventid == 'cowrie.client.var':
m['fields'].update({
'name': entry['name'],
'value': entry['value'],
})
elif eventid == 'cowrie.client.fingerprint':
m['fields'].update({
'fingerprint': entry['fingerprint']
})
# cowrie.direct-tcpip.data, cowrie.direct-tcpip.request
# cowrie.log.closed
# are not implemented
else:
# other events should be handled
log.err(
"output_influx: event '{}' not handled. Skipping..".format(
eventid))
return
result = self.client.write_points([m])
if not result:
log.err("output_influx: error when writing '{}' measurement"
"in the db.".format(eventid))
class TestInfluxDBClient(unittest.TestCase):
def setUp(self):
# By default, raise exceptions on warnings
warnings.simplefilter("error", FutureWarning)
self.cli = InfluxDBClient("localhost", 8086, "username", "password")
self.dummy_points = [
{
"measurement": "cpu_load_short",
"tags": {"host": "server01", "region": "us-west"},
"time": "2009-11-10T23:00:00.123456Z",
"fields": {"value": 0.64},
}
]
self.dsn_string = "influxdb://*****:*****@my.host.fr:1886/db"
def test_scheme(self):
cli = InfluxDBClient("host", 8086, "username", "password", "database")
self.assertEqual("http://host:8086", cli._baseurl)
cli = InfluxDBClient("host", 8086, "username", "password", "database", ssl=True)
self.assertEqual("https://host:8086", cli._baseurl)
def test_dsn(self):
cli = InfluxDBClient.from_DSN("influxdb://192.168.0.1:1886")
self.assertEqual("http://192.168.0.1:1886", cli._baseurl)
cli = InfluxDBClient.from_DSN(self.dsn_string)
self.assertEqual("http://my.host.fr:1886", cli._baseurl)
self.assertEqual("uSr", cli._username)
self.assertEqual("pWd", cli._password)
self.assertEqual("db", cli._database)
self.assertFalse(cli.use_udp)
cli = InfluxDBClient.from_DSN("udp+" + self.dsn_string)
self.assertTrue(cli.use_udp)
cli = InfluxDBClient.from_DSN("https+" + self.dsn_string)
self.assertEqual("https://my.host.fr:1886", cli._baseurl)
cli = InfluxDBClient.from_DSN("https+" + self.dsn_string, **{"ssl": False})
self.assertEqual("http://my.host.fr:1886", cli._baseurl)
def test_switch_database(self):
cli = InfluxDBClient("host", 8086, "username", "password", "database")
cli.switch_database("another_database")
self.assertEqual("another_database", cli._database)
def test_switch_user(self):
cli = InfluxDBClient("host", 8086, "username", "password", "database")
cli.switch_user("another_username", "another_password")
self.assertEqual("another_username", cli._username)
self.assertEqual("another_password", cli._password)
def test_write(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST, "http://localhost:8086/write", status_code=204)
cli = InfluxDBClient(database="db")
cli.write(
{
"database": "mydb",
"retentionPolicy": "mypolicy",
"points": [
{
"measurement": "cpu_load_short",
"tags": {"host": "server01", "region": "us-west"},
"time": "2009-11-10T23:00:00Z",
"fields": {"value": 0.64},
}
],
}
)
self.assertEqual(
m.last_request.body, b"cpu_load_short,host=server01,region=us-west " b"value=0.64 1257894000000000000\n"
)
def test_write_points(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST, "http://localhost:8086/write", status_code=204)
cli = InfluxDBClient(database="db")
cli.write_points(self.dummy_points)
self.assertEqual(
"cpu_load_short,host=server01,region=us-west " "value=0.64 1257894000123456000\n",
m.last_request.body.decode("utf-8"),
)
def test_write_points_toplevel_attributes(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST, "http://localhost:8086/write", status_code=204)
cli = InfluxDBClient(database="db")
cli.write_points(self.dummy_points, database="testdb", tags={"tag": "hello"}, retention_policy="somepolicy")
self.assertEqual(
"cpu_load_short,host=server01,region=us-west,tag=hello " "value=0.64 1257894000123456000\n",
m.last_request.body.decode("utf-8"),
)
def test_write_points_batch(self):
dummy_points = [
{
"measurement": "cpu_usage",
"tags": {"unit": "percent"},
"time": "2009-11-10T23:00:00Z",
"fields": {"value": 12.34},
},
{
"measurement": "network",
"tags": {"direction": "in"},
"time": "2009-11-10T23:00:00Z",
"fields": {"value": 123.00},
},
{
"measurement": "network",
"tags": {"direction": "out"},
"time": "2009-11-10T23:00:00Z",
"fields": {"value": 12.00},
},
]
expected_last_body = "network,direction=out,host=server01,region=us-west " "value=12.0 1257894000000000000\n"
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST, "http://localhost:8086/write", status_code=204)
cli = InfluxDBClient(database="db")
cli.write_points(
points=dummy_points, database="db", tags={"host": "server01", "region": "us-west"}, batch_size=2
)
self.assertEqual(m.call_count, 2)
self.assertEqual(expected_last_body, m.last_request.body.decode("utf-8"))
def test_write_points_udp(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(("0.0.0.0", port))
cli = InfluxDBClient("localhost", 8086, "root", "root", "test", use_udp=True, udp_port=port)
cli.write_points(self.dummy_points)
received_data, addr = s.recvfrom(1024)
self.assertEqual(
"cpu_load_short,host=server01,region=us-west " "value=0.64 1257894000123456000\n", received_data.decode()
)
def test_write_bad_precision_udp(self):
cli = InfluxDBClient("localhost", 8086, "root", "root", "test", use_udp=True, udp_port=4444)
with self.assertRaisesRegexp(Exception, "InfluxDB only supports seconds precision for udp writes"):
cli.write_points(self.dummy_points, time_precision="ms")
@raises(Exception)
def test_write_points_fails(self):
cli = InfluxDBClient("host", 8086, "username", "password", "db")
with _mocked_session(cli, "post", 500):
cli.write_points([])
def test_write_points_with_precision(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST, "http://localhost:8086/write", status_code=204)
cli = InfluxDBClient(database="db")
cli.write_points(self.dummy_points, time_precision="n")
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west " b"value=0.64 1257894000123456000\n", m.last_request.body
)
cli.write_points(self.dummy_points, time_precision="u")
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west " b"value=0.64 1257894000123456\n", m.last_request.body
)
cli.write_points(self.dummy_points, time_precision="ms")
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west " b"value=0.64 1257894000123\n", m.last_request.body
)
cli.write_points(self.dummy_points, time_precision="s")
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west " b"value=0.64 1257894000\n", m.last_request.body
)
cli.write_points(self.dummy_points, time_precision="m")
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west " b"value=0.64 20964900\n", m.last_request.body
)
cli.write_points(self.dummy_points, time_precision="h")
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west " b"value=0.64 349415\n", m.last_request.body
)
def test_write_points_bad_precision(self):
cli = InfluxDBClient()
with self.assertRaisesRegexp(
Exception, "Invalid time precision is given. " "\(use 'n', 'u', 'ms', 's', 'm' or 'h'\)"
):
cli.write_points(self.dummy_points, time_precision="g")
@raises(Exception)
def test_write_points_with_precision_fails(self):
cli = InfluxDBClient("host", 8086, "username", "password", "db")
with _mocked_session(cli, "post", 500):
cli.write_points_with_precision([])
def test_query(self):
example_response = (
'{"results": [{"series": [{"measurement": "sdfsdfsdf", '
'"columns": ["time", "value"], "values": '
'[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": '
'[{"measurement": "cpu_load_short", "columns": ["time", "value"], '
'"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://*****:*****@unittest.skip("Not implemented for 0.9")
def test_query_chunked(self):
cli = InfluxDBClient(database="db")
example_object = {
"points": [
[1415206250119, 40001, 667],
[1415206244555, 30001, 7],
[1415206228241, 20001, 788],
[1415206212980, 10001, 555],
[1415197271586, 10001, 23],
],
"measurement": "foo",
"columns": ["time", "sequence_number", "val"],
}
example_response = json.dumps(example_object) + json.dumps(example_object)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/db/db/series", text=example_response)
self.assertListEqual(cli.query("select * from foo", chunked=True), [example_object, example_object])
@raises(Exception)
def test_query_fail(self):
with _mocked_session(self.cli, "get", 401):
self.cli.query("select column_one from foo;")
def test_create_database(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text='{"results":[{}]}')
self.cli.create_database("new_db")
self.assertEqual(m.last_request.qs["q"][0], 'create database "new_db"')
def test_create_numeric_named_database(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text='{"results":[{}]}')
self.cli.create_database("123")
self.assertEqual(m.last_request.qs["q"][0], 'create database "123"')
@raises(Exception)
def test_create_database_fails(self):
with _mocked_session(self.cli, "post", 401):
self.cli.create_database("new_db")
def test_drop_database(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text='{"results":[{}]}')
self.cli.drop_database("new_db")
self.assertEqual(m.last_request.qs["q"][0], 'drop database "new_db"')
def test_drop_numeric_named_database(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text='{"results":[{}]}')
self.cli.drop_database("123")
self.assertEqual(m.last_request.qs["q"][0], 'drop database "123"')
@raises(Exception)
def test_drop_database_fails(self):
cli = InfluxDBClient("host", 8086, "username", "password", "db")
with _mocked_session(cli, "delete", 401):
cli.drop_database("old_db")
def test_get_list_database(self):
data = {
"results": [
{"series": [{"name": "databases", "values": [["new_db_1"], ["new_db_2"]], "columns": ["name"]}]}
]
}
with _mocked_session(self.cli, "get", 200, json.dumps(data)):
self.assertListEqual(self.cli.get_list_database(), [{"name": "new_db_1"}, {"name": "new_db_2"}])
@raises(Exception)
def test_get_list_database_fails(self):
cli = InfluxDBClient("host", 8086, "username", "password")
with _mocked_session(cli, "get", 401):
cli.get_list_database()
def test_get_list_series(self):
example_response = (
'{"results": [{"series": [{"name": "cpu_load_short", "columns": '
'["_id", "host", "region"], "values": '
'[[1, "server01", "us-west"]]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.assertListEqual(
self.cli.get_list_series(),
[{"name": "cpu_load_short", "tags": [{"host": "server01", "_id": 1, "region": "us-west"}]}],
)
def test_create_retention_policy_default(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.cli.create_retention_policy("somename", "1d", 4, default=True, database="db")
self.assertEqual(
m.last_request.qs["q"][0], "create retention policy somename on " "db duration 1d replication 4 default"
)
def test_create_retention_policy(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.cli.create_retention_policy("somename", "1d", 4, database="db")
self.assertEqual(
m.last_request.qs["q"][0], "create retention policy somename on " "db duration 1d replication 4"
)
def test_alter_retention_policy(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
# Test alter duration
self.cli.alter_retention_policy("somename", "db", duration="4d")
self.assertEqual(m.last_request.qs["q"][0], "alter retention policy somename on db duration 4d")
# Test alter replication
self.cli.alter_retention_policy("somename", "db", replication=4)
self.assertEqual(m.last_request.qs["q"][0], "alter retention policy somename on db replication 4")
# Test alter default
self.cli.alter_retention_policy("somename", "db", default=True)
self.assertEqual(m.last_request.qs["q"][0], "alter retention policy somename on db default")
@raises(Exception)
def test_alter_retention_policy_invalid(self):
cli = InfluxDBClient("host", 8086, "username", "password")
with _mocked_session(cli, "get", 400):
self.cli.alter_retention_policy("somename", "db")
def test_get_list_retention_policies(self):
example_response = (
'{"results": [{"series": [{"values": [["fsfdsdf", "24h0m0s", 2]],'
' "columns": ["name", "duration", "replicaN"]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://*****:*****@mock.patch("requests.Session.request")
def test_request_retry(self, mock_request):
"""Tests that two connection errors will be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 3:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 204
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database="db")
cli.write_points(self.dummy_points)
@mock.patch("requests.Session.request")
def test_request_retry_raises(self, mock_request):
"""Tests that three connection errors will not be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 4:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database="db")
with self.assertRaises(requests.exceptions.ConnectionError):
cli.write_points(self.dummy_points)
def test_get_list_users(self):
example_response = '{"results":[{"series":[{"columns":["user","admin"],' '"values":[["test",false]]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.assertListEqual(self.cli.get_list_users(), [{"user": "******", "admin": False}])
def test_get_list_users_empty(self):
example_response = '{"results":[{"series":[{"columns":["user","admin"]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.assertListEqual(self.cli.get_list_users(), [])
def test_revoke_admin_privileges(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.cli.revoke_admin_privileges("test")
self.assertEqual(m.last_request.qs["q"][0], "revoke all privileges from test")
@raises(Exception)
def test_revoke_admin_privileges_invalid(self):
cli = InfluxDBClient("host", 8086, "username", "password")
with _mocked_session(cli, "get", 400):
self.cli.revoke_admin_privileges("")
def test_grant_privilege(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.cli.grant_privilege("read", "testdb", "test")
self.assertEqual(m.last_request.qs["q"][0], "grant read on testdb to test")
@raises(Exception)
def test_grant_privilege_invalid(self):
cli = InfluxDBClient("host", 8086, "username", "password")
with _mocked_session(cli, "get", 400):
self.cli.grant_privilege("", "testdb", "test")
def test_revoke_privilege(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET, "http://localhost:8086/query", text=example_response)
self.cli.revoke_privilege("read", "testdb", "test")
self.assertEqual(m.last_request.qs["q"][0], "revoke read on testdb from test")
@raises(Exception)
def test_revoke_privilege_invalid(self):
cli = InfluxDBClient("host", 8086, "username", "password")
with _mocked_session(cli, "get", 400):
self.cli.revoke_privilege("", "testdb", "test")
class TestInfluxDBClient(unittest.TestCase):
def setUp(self):
# By default, raise exceptions on warnings
warnings.simplefilter('error', FutureWarning)
self.cli = InfluxDBClient('localhost', 8086, 'username', 'password')
self.dummy_points = [
{
"name": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"timestamp": "2009-11-10T23:00:00Z",
"fields": {
"value": 0.64
}
}
]
def test_scheme(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
assert cli._baseurl == 'http://*****:*****@host:1886/db')
assert cli._baseurl == 'http://*****:*****@host:1886/db')
assert cli.use_udp is True
cli = InfluxDBClient.from_DSN('https+influxdb://usr:pwd@host:1886/db')
assert cli._baseurl == 'https://*****:*****@host:1886/db',
**{'ssl': False})
assert cli._baseurl == 'http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_write_points_batch(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 200, self.dummy_points):
assert cli.write_points(
data=self.dummy_points,
batch_size=2
) is True
def test_write_points_udp(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient(
'localhost', 8086, 'root', 'root',
'test', use_udp=True, udp_port=port
)
cli.write_points(self.dummy_points)
received_data, addr = s.recvfrom(1024)
self.assertDictEqual(
{
"points": self.dummy_points,
"database": "test"
},
json.loads(received_data.decode(), strict=True)
)
def test_write_bad_precision_udp(self):
cli = InfluxDBClient(
'localhost', 8086, 'root', 'root',
'test', use_udp=True, udp_port=4444
)
with self.assertRaisesRegexp(
Exception,
"InfluxDB only supports seconds precision for udp writes"
):
cli.write_points(
self.dummy_points,
time_precision='ms'
)
@raises(Exception)
def test_write_points_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points([])
def test_write_points_with_precision(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write"
)
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points,
time_precision='n'
)
self.assertDictEqual(
{'points': self.dummy_points,
'database': 'db',
'precision': 'n',
},
json.loads(m.last_request.body)
)
def test_write_points_bad_precision(self):
cli = InfluxDBClient()
with self.assertRaisesRegexp(
Exception,
"Invalid time precision is given. "
"\(use 'n', 'u', 'ms', 's', 'm' or 'h'\)"
):
cli.write_points(
self.dummy_points,
time_precision='g'
)
@raises(Exception)
def test_write_points_with_precision_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points_with_precision([])
def test_query(self):
example_response = \
'{"results": [{"series": [{"name": "sdfsdfsdf", ' \
'"columns": ["time", "value"], "values": ' \
'[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": ' \
'[{"name": "cpu_load_short", "columns": ["time", "value"], ' \
'"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_query_chunked(self):
cli = InfluxDBClient(database='db')
example_object = {
'points': [
[1415206250119, 40001, 667],
[1415206244555, 30001, 7],
[1415206228241, 20001, 788],
[1415206212980, 10001, 555],
[1415197271586, 10001, 23]
],
'name': 'foo',
'columns': [
'time',
'sequence_number',
'val'
]
}
example_response = \
json.dumps(example_object) + json.dumps(example_object)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/db/db/series",
text=example_response
)
self.assertListEqual(
cli.query('select * from foo', chunked=True),
[example_object, example_object]
)
@raises(Exception)
def test_query_fail(self):
with _mocked_session(self.cli, 'get', 401):
self.cli.query('select column_one from foo;')
def test_create_database(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.create_database('new_db')
self.assertEqual(
m.last_request.qs['q'][0],
'create database new_db'
)
@raises(Exception)
def test_create_database_fails(self):
with _mocked_session(self.cli, 'post', 401):
self.cli.create_database('new_db')
def test_drop_database(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.drop_database('new_db')
self.assertEqual(
m.last_request.qs['q'][0],
'drop database new_db'
)
@raises(Exception)
def test_drop_database_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'delete', 401):
cli.drop_database('old_db')
def test_get_list_database(self):
data = {'results': [
{'series': [
{'name': 'databases',
'values': [
['new_db_1'],
['new_db_2']],
'columns': ['name']}]}
]}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(
self.cli.get_list_database(),
[{'name': 'new_db_1'}, {'name': 'new_db_2'}]
)
@raises(Exception)
def test_get_list_database_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_database()
def test_get_list_series(self):
example_response = \
'{"results": [{"series": [{"name": "cpu_load_short", "columns": ' \
'["_id", "host", "region"], "values": ' \
'[[1, "server01", "us-west"]]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://*****:*****@mock.patch('requests.Session.request')
def test_request_retry(self, mock_request):
"""Tests that two connection errors will be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 3:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points
)
@mock.patch('requests.Session.request')
def test_request_retry_raises(self, mock_request):
"""Tests that three connection errors will not be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 4:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
with self.assertRaises(requests.exceptions.ConnectionError):
cli.write_points(self.dummy_points)
class Tourbillon(object):
"""create a tourbillon instance reading its configuration from config_file
"""
def __init__(self, config_file):
super(Tourbillon, self).__init__()
self._aio_run_event = asyncio.Event()
self._thr_run_event = threading.Event()
self._loop = asyncio.get_event_loop()
self._tasks = []
self._pluginconfig = {}
with open(config_file, 'r') as f:
self._config = json.load(f)
formatter = logging.Formatter(fmt=self._config['log_format'])
handler = logging.handlers.WatchedFileHandler(
self._config['log_file'])
handler.setFormatter(formatter)
handler.setLevel(getattr(logging, self._config['log_level']))
logging.getLogger().addHandler(handler)
logging.getLogger().setLevel(
getattr(logging, self._config['log_level']))
logger.info('Use config file: %s', config_file)
self._load_plugins_config(os.path.abspath(
os.path.dirname(config_file)))
self._influxdb = InfluxDBClient(**self._config['database'])
self._databases = [i['name']
for i in self._influxdb.get_list_database()]
print(self._databases)
def _load_plugins_config(self, tourbillon_conf_dir):
t = Template(self._config['plugins_conf_dir'])
plugin_conf_dir = t.safe_substitute(
tourbillon_conf_dir=tourbillon_conf_dir)
logger.info('Plugin config dir: %s', plugin_conf_dir)
config_files = glob.glob(os.path.join(plugin_conf_dir,
'*.conf'))
for file_name in config_files:
k = os.path.splitext(os.path.basename(file_name))[0]
with open(file_name, 'r') as f:
try:
self._pluginconfig[k] = json.load(f)
except:
logger.exception('error loading config file %s', file_name)
@property
def config(self):
"""returns a dictionary that contains configuration for each enabled
plugin"""
return self._pluginconfig
@property
def run_event(self):
"""get the asyncio.Event or threading.Event"""
cf = inspect.currentframe()
caller_name = cf.f_back.f_code.co_name
caller = cf.f_back.f_globals[caller_name]
if asyncio.iscoroutinefunction(caller) or asyncio.iscoroutine(caller):
return self._aio_run_event
else:
return self._thr_run_event
def push(self, points, database):
"""write syncronously datapoints to InfluxDB"""
self._influxdb.write_points(points, database=database)
def create_database(self, name, duration=None, replication=None,
default=True):
"""create syncronously a database and a retention policy
in the InfluxDB instance"""
if name not in self._databases:
self._influxdb.create_database(name)
logger.info('database %s created successfully', name)
if duration and replication:
rps = self._influxdb.get_list_retention_policies(name)
tourbillon_rp_name = '%s_tourbillon' % name
duration_in_hours = _to_hours(duration)
logger.debug('duration_in_hours: %s', duration_in_hours)
for rp in rps:
if rp['name'] == tourbillon_rp_name:
logger.debug('current rp config: %s', rp)
if rp['duration'] == duration_in_hours and \
rp['replicaN'] == int(replication) and \
rp['default'] == default:
logger.debug('the retention policy %s already exists',
tourbillon_rp_name)
return
self._influxdb.alter_retention_policy(
tourbillon_rp_name,
database=name,
duration=duration,
replication=replication,
default=default
)
logger.info('retention policy %s altered successfully',
tourbillon_rp_name)
return
self._influxdb.create_retention_policy(
tourbillon_rp_name,
database=name,
duration=duration,
replication=replication,
default=default
)
logger.info('retention policy %s created successfully',
tourbillon_rp_name)
@asyncio.coroutine
def async_push(self, points, database):
"""write asyncronously datapoints to InfluxDB"""
yield From(self._loop.run_in_executor(
None,
functools.partial(self._influxdb.write_points,
points, database=database)))
@asyncio.coroutine
def async_create_database(self, name, duration=None, replication=None,
default=True):
"""create asyncronously a database and a retention policy
in the InfluxDB instance"""
if name not in self._databases:
yield From(self._loop.run_in_executor(
None,
self._influxdb.create_database,
name))
logger.info('database %s created successfully', name)
if duration and replication:
rps = yield From(self._loop.run_in_executor(
None,
self._influxdb.get_list_retention_policies,
name))
tourbillon_rp_name = '%s_tourbillon' % name
duration_in_hours = _to_hours(duration)
logger.debug('duration_in_hours: %s', duration_in_hours)
for rp in rps:
if rp['name'] == tourbillon_rp_name:
logger.debug('current rp: %s', rp)
if rp['duration'] == duration_in_hours and \
rp['replicaN'] == int(replication) and \
rp['default'] == default:
logger.debug('the retention policy %s already exists',
tourbillon_rp_name)
return
yield From(self._loop.run_in_executor(
None,
functools.partial(
self._influxdb.alter_retention_policy,
tourbillon_rp_name,
database=name,
duration=duration,
replication=replication,
default=default
)
))
logger.info('retention policy %s altered successfully',
tourbillon_rp_name)
return
yield From(self._loop.run_in_executor(
None,
functools.partial(
self._influxdb.create_retention_policy,
tourbillon_rp_name,
database=name,
duration=duration,
replication=replication,
default=default
)
))
logger.info('retention policy %s created successfully',
tourbillon_rp_name)
def _load_tasks(self):
if 'plugins' not in self._config:
logger.warn('no plugin configured.')
return
plugins = self._config['plugins']
thread_targets_count = 0
for module_name, functions in plugins.items():
logger.debug('search for tasks in module %s', module_name)
module = import_module(module_name)
logger.debug('module %s successfully imported', module_name)
for task_name in functions:
logger.debug('checking declared task %s', task_name)
if hasattr(module, task_name):
candidate_task = getattr(module, task_name)
task_type = ''
if asyncio.iscoroutinefunction(candidate_task):
self._tasks.append(asyncio.async(
candidate_task(self)))
task_type = 'coroutine'
else:
self._tasks.append(self._loop.run_in_executor(
None,
candidate_task,
self))
task_type = 'function'
thread_targets_count += 1
logger.info('task found: %s.%s, type=%s',
module_name, task_name, task_type)
if thread_targets_count > 0:
self._loop.set_default_executor(ThreadPoolExecutor(
max_workers=thread_targets_count + 2)
)
logger.debug('configured tasks: %s', self._tasks)
def stop(self):
"""stop the tourbillon agent"""
self._loop.remove_signal_handler(signal.SIGINT)
self._loop.remove_signal_handler(signal.SIGTERM)
logger.info('shutting down tourbillon...')
self._aio_run_event.clear()
self._thr_run_event.clear()
def run(self):
"""start the tourbillon agent"""
logger.info('starting tourbillon...')
self._loop.add_signal_handler(signal.SIGINT, self.stop)
self._loop.add_signal_handler(signal.SIGTERM, self.stop)
self._load_tasks()
self._aio_run_event.set()
self._thr_run_event.set()
logger.info('tourbillon started')
self._loop.run_until_complete(asyncio.wait(self._tasks))
logger.info('tourbillon shutdown completed')
def main(host='localhost', port=8086):
user = '******'
password = '******'
dbname = 'example'
dbuser = '******'
dbuser_password = '******'
query = 'select value from cpu_load_short;'
"""
json_body = [
{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"value": 0.64
}
}
]
"""
"""
json_body = [
{'fields': {
'value': 0.0
},
'tags': {
'source': 'wago',
'site_name': 'Nyange'
},
'time': '2015-11-05T07:39:41Z',
'measurement': 'rTotalApparentPower'}
]
"""
json_body = [{'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalActiveEnergy'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalReactiveEnergy'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'DCPower2'}, {'fields': {'value': 2.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'DCPower1'}, {'fields': {'value': 9.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'DCVolt1'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arVoltage_L_N2'}, {'fields': {'value': 9.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arVoltage_L_N1'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'DCVolt2'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arVoltage_L_N3'}, {'fields': {'value': 0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'xRotatingField'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalApparentEnergy'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arFrequency3'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arFrequency2'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arFrequency1'}, {'fields': {'value': datetime.datetime(2015, 11, 5, 7, 39, 41)}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'timestamp'}, {'fields': {'value': 3.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalActivePower'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arCurrent2'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arCurrent3'}, {'fields': {'value': 9.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'arCurrent1'}, {'fields': {'value': 1000L}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'iSiteCode'}, {'fields': {'value': 3.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalReactivePower'}, {'fields': {'value': 0L}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'trans'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'DCCurr2'}, {'fields': {'value': 10.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'DCCurr1'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalPowerFactorPF'}, {'fields': {'value': 0.0}, 'tags': {'source': 'wago', 'site_name': 'Nyange'}, 'time': '2015-11-05T07:39:41Z', 'measurement': 'rTotalApparentPower'}]
client = InfluxDBClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
print("Switch user: "******"Write points: {0}".format(json_body))
client.write_points(json_body)
print("Queying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print("Switch user: "******"Drop database: " + dbname)
client.drop_database(dbname)
def send_to_influx(self):
if not self.config['hostname']:
self.log.error("No Influx server configured, please set one using: "
"ceph influx config-set hostname <hostname>")
self.set_health_checks({
'MGR_INFLUX_NO_SERVER': {
'severity': 'warning',
'summary': 'No InfluxDB server configured',
'detail': ['Configuration option hostname not set']
}
})
return
# If influx server has authentication turned off then
# missing username/password is valid.
self.log.debug("Sending data to Influx host: %s",
self.config['hostname'])
client = InfluxDBClient(self.config['hostname'], self.config['port'],
self.config['username'],
self.config['password'],
self.config['database'],
self.config['ssl'],
self.config['verify_ssl'])
# using influx client get_list_database requires admin privs,
# instead we'll catch the not found exception and inform the user if
# db can not be created
try:
df_stats, pools = self.get_df_stats()
client.write_points(df_stats, 'ms')
client.write_points(self.get_daemon_stats(), 'ms')
client.write_points(self.get_pg_summary(pools))
self.set_health_checks(dict())
except ConnectionError as e:
self.log.exception("Failed to connect to Influx host %s:%d",
self.config['hostname'], self.config['port'])
self.set_health_checks({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB server at %s:%d'
' due to an connection error'
% (self.config['hostname'], self.config['port']),
'detail': [str(e)]
}
})
except InfluxDBClientError as e:
if e.code == 404:
self.log.info("Database '%s' not found, trying to create "
"(requires admin privs). You can also create "
"manually and grant write privs to user "
"'%s'", self.config['database'],
self.config['username'])
client.create_database(self.config['database'])
client.create_retention_policy(name='8_weeks', duration='8w',
replication='1', default=True,
database=self.config['database'])
else:
self.set_health_checks({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB',
'detail': [str(e)]
}
})
raise
class InfluxDBExporter(object):
def __init__(self, host, port,
dbname, user, pwd,
db_management, geohash={}):
self.host = host
self.port = port
self.dbname = dbname
self.user = user
self.pwd = pwd
self.client = None
self.geohash = geohash
self.NB_MAX_TRY_REQUEST = 10 # nb of rqt error before aborting
# self.TIME_MAX = 1*60.*60.
# add one item by influxdb line
self.data = []
self.nb_data_max = 40000 # no more than 5000 (cf. influxdb doc.)
self.client = InfluxDBClient(host=host, port=port, database=dbname)
if db_management:
self.prepare_db(db_management)
def prepare_db(self, db_management):
if db_management['drop_db']:
self.drop_db()
self.create_db()
self.set_retention_policies(db_management['retention'])
def drop_db(self, dbname=None):
if not dbname:
dbname = self.dbname
logger.info("Drop %s database." % dbname)
try:
self.client.drop_database(dbname)
except:
logger.info("Can't drop %s database (not existing yet ?)."
% dbname)
def create_db(self, dbname=None):
if not dbname:
dbname = self.dbname
logger.info("Open/Create %s database." % dbname)
# self.client.create_database(dbname, if_not_exists=True)
self.client.create_database(dbname)
self.client.switch_database(dbname)
def set_retention_policies(self, days, dbname=None):
if not dbname:
dbname = self.dbname
name = "in_days"
logger.info("Setting %s retention policy on %s database, keep=%d days."
% (name, dbname, days))
try:
self.client.create_retention_policy(name,
duration="%dd" % days,
replication="1",
database=dbname, default=True)
except:
self.client.alter_retention_policy(name,
database=dbname,
duration="%dd" % days,
replication=1,
default=True)
def make_stats(self, now):
t = timegm(now.utctimetuple()) * 1e9 \
+ now.microsecond * 1e3
t_str = str(int(t))
s = "queue,type=producer size=%d " % q.qsize() + t_str
self.data.append(s)
s = "queue,type=consumer size=%d " % len(self.data) + t_str
self.data.append(s)
def make_line_latency(self, channel, starttime, latency_value):
timestamp = starttime.datetime
t = timegm(timestamp.utctimetuple()) * 1e9 \
+ timestamp.microsecond * 1e3
try:
geohash_tag = ",geohash=%s" % self.geohash[channel]
except:
geohash_tag = ""
l = "latency,channel=" + channel + \
geohash_tag + \
" value=" + "%.1f " % latency_value + \
str(int(t))
self.data.append(l)
def make_line_count(self, channel, starttime, delta, data):
cc = "count,channel=" + channel
for i, v in enumerate(data):
timestamp = starttime + i*delta
t = timegm(timestamp.utctimetuple()) * 1e9 \
+ timestamp.microsecond * 1e3
c = cc + " value=" + "%.2f " % v + str(int(t))
self.data.append(c)
def send_points(self, debug=False):
"""Send points to influxsb
to speed-up things make our own "data line"
(bypass influxdb write_points python api)
"""
data = '\n'.join(self.data[:self.nb_data_max])
del self.data[:self.nb_data_max]
headers = self.client._headers
headers['Content-type'] = 'application/octet-stream'
nb_try = 0
while True:
nb_try += 1
try:
self.client.request(url="write",
method='POST',
params={'db': self.client._database},
data=data,
expected_response_code=204,
headers=headers
)
except (InfluxDBServerError,
InfluxDBClientError,
requests.exceptions.ConnectionError) as e:
if nb_try > self.NB_MAX_TRY_REQUEST:
raise e
else:
logger.error("Request failed (%s)" % e)
logger.error("retrying (%d/%d)" %
(nb_try, self.NB_MAX_TRY_REQUEST))
continue
break
def manage_data(self, trace):
"""Return True is data have been pushed to influxdb"""
delta = trace.stats['delta']
starttime = trace.stats['starttime']
channel = trace.get_id()
now = datetime.utcnow()
nbsamples = len(trace.data)
last_sample_time = starttime + delta * (nbsamples - 1)
l = UTCDateTime(now) - last_sample_time
lock.acquire()
last_packet_time[channel] = last_sample_time
lock.release()
# do not process 'old' data
# if l > self.TIME_MAX:
# return
self.make_line_count(channel,
starttime,
delta,
trace.data)
self.make_line_latency(channel,
starttime + delta * (nbsamples - 1),
l)
# send data to influxdb if buffer is filled enough
if len(self.data) > self.nb_data_max:
now = datetime.utcnow()
self.make_stats(now)
logger.debug("Data sent")
try:
self.send_points(debug=False)
except InfluxDBServerError as e:
self.force_shutdown(e)
else:
return True
else:
return False
def run(self):
"""Run unless shutdown signal is received. """
# time in seconds
timeout = 0.1
max_cumulated_wait_time = 15
wait_time = 0
while True:
try:
trace = q.get(timeout=timeout)
except Queue.Empty:
# process queue before shutdown
if q.empty() and shutdown_event.isSet():
logger.info("%s thread has catched *shutdown_event*" %
self.__class__.__name__)
sys.exit(0)
wait_time += timeout
if wait_time > max_cumulated_wait_time:
# force data flush to influxdb
# even if data block is not completed
logger.info('Timer reached (%ds)' % max_cumulated_wait_time
+ '. Force data flush to influxdb '
+ '(bsize=%d/%d)!'
% (len(self.data), self.nb_data_max))
now = datetime.utcnow()
self.make_stats(now)
try:
self.send_points()
except BaseException as e:
self.force_shutdown(e)
wait_time = 0
else:
data_pushed = self.manage_data(trace)
q.task_done()
if data_pushed:
wait_time = 0
class TestInfluxDBClient(unittest.TestCase):
def setUp(self):
# By default, raise exceptions on warnings
warnings.simplefilter('error', FutureWarning)
self.cli = InfluxDBClient('localhost', 8086, 'username', 'password')
self.dummy_points = [
{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00.123456Z",
"fields": {
"value": 0.64
}
}
]
self.dsn_string = 'influxdb://*****:*****@host:1886/db'
def test_scheme(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
self.assertEqual('http://host:8086', cli._baseurl)
cli = InfluxDBClient(
'host', 8086, 'username', 'password', 'database', ssl=True
)
self.assertEqual('https://host:8086', cli._baseurl)
def test_dsn(self):
cli = InfluxDBClient.from_DSN(self.dsn_string)
self.assertEqual('http://host:1886', cli._baseurl)
self.assertEqual('uSr', cli._username)
self.assertEqual('pWd', cli._password)
self.assertEqual('db', cli._database)
self.assertFalse(cli.use_udp)
cli = InfluxDBClient.from_DSN('udp+' + self.dsn_string)
self.assertTrue(cli.use_udp)
cli = InfluxDBClient.from_DSN('https+' + self.dsn_string)
self.assertEqual('https://host:1886', cli._baseurl)
cli = InfluxDBClient.from_DSN('https+' + self.dsn_string,
**{'ssl': False})
self.assertEqual('http://host:1886', cli._baseurl)
def test_switch_database(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
cli.switch_database('another_database')
self.assertEqual('another_database', cli._database)
def test_switch_user(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
cli.switch_user('another_username', 'another_password')
self.assertEqual('another_username', cli._username)
self.assertEqual('another_password', cli._password)
def test_write(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write(
{"database": "mydb",
"retentionPolicy": "mypolicy",
"points": [{"measurement": "cpu_load_short",
"tags": {"host": "server01",
"region": "us-west"},
"time": "2009-11-10T23:00:00Z",
"fields": {"value": 0.64}}]}
)
self.assertEqual(
m.last_request.body,
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000000000000\n",
)
def test_write_points(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points,
)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west '
'value=0.64 1257894000123456000\n',
m.last_request.body.decode('utf-8'),
)
def test_write_points_toplevel_attributes(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points,
database='testdb',
tags={"tag": "hello"},
retention_policy="somepolicy"
)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west,tag=hello '
'value=0.64 1257894000123456000\n',
m.last_request.body.decode('utf-8'),
)
def test_write_points_batch(self):
dummy_points = [
{"measurement": "cpu_usage", "tags": {"unit": "percent"},
"time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}},
{"measurement": "network", "tags": {"direction": "in"},
"time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}},
{"measurement": "network", "tags": {"direction": "out"},
"time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}}
]
expected_last_body = (
"network,direction=out,host=server01,region=us-west "
"value=12.0 1257894000000000000\n"
)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write_points(points=dummy_points,
database='db',
tags={"host": "server01",
"region": "us-west"},
batch_size=2)
self.assertEqual(m.call_count, 2)
self.assertEqual(expected_last_body,
m.last_request.body.decode('utf-8'))
def test_write_points_udp(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient(
'localhost', 8086, 'root', 'root',
'test', use_udp=True, udp_port=port
)
cli.write_points(self.dummy_points)
received_data, addr = s.recvfrom(1024)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west '
'value=0.64 1257894000123456000\n',
received_data.decode()
)
def test_write_bad_precision_udp(self):
cli = InfluxDBClient(
'localhost', 8086, 'root', 'root',
'test', use_udp=True, udp_port=4444
)
with self.assertRaisesRegexp(
Exception,
"InfluxDB only supports seconds precision for udp writes"
):
cli.write_points(
self.dummy_points,
time_precision='ms'
)
@raises(Exception)
def test_write_points_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points([])
def test_write_points_with_precision(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.POST,
"http://localhost:8086/write",
status_code=204
)
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points, time_precision='n')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456000\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='u')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='ms')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='s')
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000\n",
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='m')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 20964900\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='h')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 349415\n',
m.last_request.body,
)
def test_write_points_bad_precision(self):
cli = InfluxDBClient()
with self.assertRaisesRegexp(
Exception,
"Invalid time precision is given. "
"\(use 'n', 'u', 'ms', 's', 'm' or 'h'\)"
):
cli.write_points(
self.dummy_points,
time_precision='g'
)
@raises(Exception)
def test_write_points_with_precision_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points_with_precision([])
def test_query(self):
example_response = (
'{"results": [{"series": [{"measurement": "sdfsdfsdf", '
'"columns": ["time", "value"], "values": '
'[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": '
'[{"measurement": "cpu_load_short", "columns": ["time", "value"], '
'"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_query_chunked(self):
cli = InfluxDBClient(database='db')
example_object = {
'points': [
[1415206250119, 40001, 667],
[1415206244555, 30001, 7],
[1415206228241, 20001, 788],
[1415206212980, 10001, 555],
[1415197271586, 10001, 23]
],
'measurement': 'foo',
'columns': [
'time',
'sequence_number',
'val'
]
}
example_response = \
json.dumps(example_object) + json.dumps(example_object)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/db/db/series",
text=example_response
)
self.assertListEqual(
cli.query('select * from foo', chunked=True),
[example_object, example_object]
)
@raises(Exception)
def test_query_fail(self):
with _mocked_session(self.cli, 'get', 401):
self.cli.query('select column_one from foo;')
def test_create_database(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.create_database('new_db')
self.assertEqual(
m.last_request.qs['q'][0],
'create database new_db'
)
@raises(Exception)
def test_create_database_fails(self):
with _mocked_session(self.cli, 'post', 401):
self.cli.create_database('new_db')
def test_drop_database(self):
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}'
)
self.cli.drop_database('new_db')
self.assertEqual(
m.last_request.qs['q'][0],
'drop database new_db'
)
@raises(Exception)
def test_drop_database_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'delete', 401):
cli.drop_database('old_db')
def test_get_list_database(self):
data = {'results': [
{'series': [
{'name': 'databases',
'values': [
['new_db_1'],
['new_db_2']],
'columns': ['name']}]}
]}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(
self.cli.get_list_database(),
[{'name': 'new_db_1'}, {'name': 'new_db_2'}]
)
@raises(Exception)
def test_get_list_database_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_database()
def test_get_list_series(self):
example_response = \
'{"results": [{"series": [{"name": "cpu_load_short", "columns": ' \
'["_id", "host", "region"], "values": ' \
'[[1, "server01", "us-west"]]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.assertListEqual(
self.cli.get_list_series(),
[{'name': 'cpu_load_short',
'tags': [
{'host': 'server01', '_id': 1, 'region': 'us-west'}
]}]
)
def test_create_retention_policy_default(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.cli.create_retention_policy(
'somename', '1d', 4, default=True, database='db'
)
self.assertEqual(
m.last_request.qs['q'][0],
'create retention policy somename on '
'db duration 1d replication 4 default'
)
def test_create_retention_policy(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.cli.create_retention_policy(
'somename', '1d', 4, database='db'
)
self.assertEqual(
m.last_request.qs['q'][0],
'create retention policy somename on '
'db duration 1d replication 4'
)
def test_alter_retention_policy(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
# Test alter duration
self.cli.alter_retention_policy('somename', 'db',
duration='4d')
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy somename on db duration 4d'
)
# Test alter replication
self.cli.alter_retention_policy('somename', 'db',
replication=4)
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy somename on db replication 4'
)
# Test alter default
self.cli.alter_retention_policy('somename', 'db',
default=True)
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy somename on db default'
)
@raises(Exception)
def test_alter_retention_policy_invalid(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.alter_retention_policy('somename', 'db')
def test_get_list_retention_policies(self):
example_response = \
'{"results": [{"series": [{"values": [["fsfdsdf", "24h0m0s", 2]],'\
' "columns": ["name", "duration", "replicaN"]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://*****:*****@mock.patch('requests.Session.request')
def test_request_retry(self, mock_request):
"""Tests that two connection errors will be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 3:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 204
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
cli.write_points(
self.dummy_points
)
@mock.patch('requests.Session.request')
def test_request_retry_raises(self, mock_request):
"""Tests that three connection errors will not be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 4:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
with self.assertRaises(requests.exceptions.ConnectionError):
cli.write_points(self.dummy_points)
def test_get_list_users(self):
example_response = (
'{"results":[{"series":[{"columns":["user","admin"],'
'"values":[["test",false]]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.assertListEqual(
self.cli.get_list_users(),
[{'user': '******', 'admin': False}]
)
def test_get_list_users_empty(self):
example_response = (
'{"results":[{"series":[{"columns":["user","admin"]}]}]}'
)
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.assertListEqual(self.cli.get_list_users(), [])
def test_revoke_admin_privileges(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.cli.revoke_admin_privileges('test')
self.assertEqual(
m.last_request.qs['q'][0],
'revoke all privileges from test'
)
@raises(Exception)
def test_revoke_admin_privileges_invalid(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.revoke_admin_privileges('')
def test_grant_privilege(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.cli.grant_privilege('read', 'testdb', 'test')
self.assertEqual(
m.last_request.qs['q'][0],
'grant read on testdb to test'
)
@raises(Exception)
def test_grant_privilege_invalid(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.grant_privilege('', 'testdb', 'test')
def test_revoke_privilege(self):
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(
requests_mock.GET,
"http://localhost:8086/query",
text=example_response
)
self.cli.revoke_privilege('read', 'testdb', 'test')
self.assertEqual(
m.last_request.qs['q'][0],
'revoke read on testdb from test'
)
@raises(Exception)
def test_revoke_privilege_invalid(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.revoke_privilege('', 'testdb', 'test')
"measurement" : "speedtest_metrics" ,
"tags": {
"host": "iron.lan",
"region": "home"
},
"fields": {
"ping": _ping,
"download" : _download,
"upload": _upload
}
}
]
client = InfluxDBClient(
os.environ['INFLUXDB_HOST'],
os.environ['INFLUXDB_PORT'],
os.environ['INFLUXDB_USER'],
os.environ['INFLUXDB_PASSWORD'],
os.environ['INFLUXDB_DATABASE']
)
client.create_database(os.environ['INFLUXDB_DATABASE'])
client.create_retention_policy("speedtest_policy", "30d",1, default=True)
# writing points
print "===> writing points"
client.write_points(json_body)
else:
print "failed to retrieve data"
