def main(host='localhost', port=8086):
"""Instantiate a connection to the InfluxDB."""
user = '******'
password = '******'
dbname = 'example'
dbuser = '******'
dbuser_password = '******'
query = 'select Float_value from cpu_load_short;'
query_where = 'select Int_value from cpu_load_short where host=$host;'
bind_params = {'host': 'server01'}
json_body = [
{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"Float_value": 0.64,
"Int_value": 3,
"String_value": "Text",
"Bool_value": True
}
}
]
client = InfluxDBClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
print("Create a retention policy")
client.create_retention_policy('awesome_policy', '3d', 3, default=True)
print("Switch user: "******"Write points: {0}".format(json_body))
client.write_points(json_body)
print("Querying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print("Querying data: " + query_where)
result = client.query(query_where, bind_params=bind_params)
print("Result: {0}".format(result))
print("Switch user: "******"Drop database: " + dbname)
client.drop_database(dbname)
class IdmConnection(object):
def __init__(self, user, pswd, host='localhost', port=8086, db='metrics'):
self.host = host
self.port = int(port)
self.client = InfluxDBClient(self.host, self.port, user, pswd, db)
self.client.create_database(db)
self.globalLabels = {}
def addGlobalLabel(self, name, value):
"""
Add one label with value into global labels.
"""
self.globalLabels[name] = value
def setGlobalLabels(self, labels, append=False):
"""
Set global labels for this metric.
Default clear previous labels, unless append is True
"""
if not append:
self.globalLabels = {}
self.globalLabels.update(labels)
def metric(self, name):
return Metric(name, self.client, self.globalLabels.copy())
def main(host='localhost', port=8086, nb_day=15):
nb_day = 15 # number of day to generate time series
timeinterval_min = 5 # create an event every x minutes
total_minutes = 1440 * nb_day
total_records = int(total_minutes / timeinterval_min)
now = datetime.datetime.today()
cpu_series = [{
'name': "server_data.cpu_idle",
'columns': ["time", "value", "hostName"],
'points': []
}]
for i in range(0, total_records):
past_date = now - datetime.timedelta(minutes=i * timeinterval_min)
value = random.randint(0, 200)
hostName = "server-%d" % random.randint(1, 5)
pointValues = [int(past_date.strftime('%s')), value, hostName]
cpu_series[0]['points'].append(pointValues)
client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)
print("Create database: " + DBNAME)
client.create_database(DBNAME)
print("Write points #: {0}".format(total_records))
client.write_points(cpu_series)
query = 'SELECT MEAN(value) FROM server_data.cpu_idle GROUP BY time(30m) WHERE time > now() - 1d;'
print("Queying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print("Delete database: " + DBNAME)
client.delete_database(DBNAME)
class InfluxDatabase(Database):
def __init__(self, props, database):
super().__init__(database)
host = props.get('host', 'localhost')
port = props.get('port', 8086)
self.client = InfluxDBClient(host=host, port=port)
self.client.create_database(self.database)
self.client.switch_database(self.database)
def now(self):
return super().now()
def _write_all_points(self, points, align_points=False):
if align_points:
time = self.now()
for p in points:
p['time'] = time
self.client.write_points(points)
def _point(self, name, fields, tags={}):
return {
'measurement': name,
'time': self.now(),
'tags': tags,
'fields': fields
}
def time_format(self):
return '%Y-%m-%dT%H:%M:%SZ'
def main(args=None):
parser = ArgumentParser(usage="Usage: %(prog)s [options]", description="query datacatalog")
parser.add_argument("-H","--host",dest='host',help="hostname of influxdb instance")
parser.add_argument("-u","--user",dest="user",help="username")
parser.add_argument("-p","--password",dest="pw",help="password")
parser.add_argument("-P","--port",dest="port",type=int,default=8086,help="influxdb ingest port")
parser.add_argument("-n","--dbname", dest="dbname",help="name of DB to store data in.")
parser.add_argument("-d", "--dry", dest="dry", action="store_true", default=False, help="do not report results to grafana")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, help="verbose mode")
opts = parser.parse_args(args)
json_bdy = []
for site in batchSites:
status_dict = {key: 0 for key in statii}
stats = JobInstance.objects.filter(site=site).item_frequencies("status")
status_dict.update(stats)
for stat, freq in status_dict.iteritems():
json_bdy.append(__makeEntry__(stat, site, freq))
print 'found %i measurements to add'%len(json_bdy)
pp = PrettyPrinter(indent=2)
if opts.verbose: pp.pprint(json_bdy)
if opts.dry:
return
if influxdb:
client = InfluxDBClient(opts.host,opts.port,opts.user,opts.pw,opts.dbname)
client.create_database(opts.dbname)
ret = client.write_points(json_bdy)
if not ret:
try:
raise Exception("Could not write points to DB")
except Exception:
print_exc()
sys_exit(int(ret))
def write(self, repository, counters,
current_time=int(time.time())):
client = InfluxDBClient(self.host, self.port, self.user, self.password, None, True, False)
print client.get_list_database()
if {"name": self.database} not in client.get_list_database():
client.create_database(self.database)
client.switch_database(self.database)
ts = time.time()
st = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%dT%H:%M:%SZ')
json_body = [{
"measurement": "time",
"tags": {
"host": "cvm-perf01",
"region": repository,
},
"time": st,
"fields": {
"value": current_time
}
}]
client.write_points(json_body)
for counter in counters.values():
json_body[0]["measurement"] = counter.name
json_body[0]["fields"]["value"] = int(counter.avg())
client.write_points(json_body)
def send_to_influx(self):
host = self.get_config("hostname")
if not host:
self.log.error("No InfluxDB server configured, please set"
"`hostname` configuration key.")
return
port = int(self.get_config("port", default="8086"))
database = self.get_config("database", default="ceph")
# If influx server has authentication turned off then
# missing username/password is valid.
username = self.get_config("username", default="")
password = self.get_config("password", default="")
client = InfluxDBClient(host, port, username, password, database)
# using influx client get_list_database requires admin privs, instead we'll catch the not found exception and inform the user if db can't be created
try:
client.write_points(self.get_df_stats(), 'ms')
client.write_points(self.get_daemon_stats(), 'ms')
except InfluxDBClientError as e:
if e.code == 404:
self.log.info("Database '{0}' not found, trying to create (requires admin privs). You can also create manually and grant write privs to user '{1}'".format(database,username))
client.create_database(database)
else:
raise
def public_metrics_into_influxdb():
ping, download, upload = get_internet_measure()
current_time = get_time()
net_json_body = [
{
"measurement": "internet_measure",
"tags": {
"host": "desktop-lab",
"contract": "Vivo Fibra"
},
"time": str(current_time),
"fields": {
"ping": ping,
"download":download,
"upload": upload
}
}
]
print ("\n=========" )
print ("Inserting: \n" + str(net_json_body))
influxdb_host = getting_env_container_hosts()
client = InfluxDBClient(influxdb_host, 8086, 'root', 'root', 'internet_measure')
client.create_database('internet_measure')
client.write_points(net_json_body)
print ("=========\n")
def connect_db(host, port, user, password, dbname):
try:
client = InfluxDBClient(host, port, user, password)
database = client.get_list_database()
''' client.get_list_database()
e.g. [{u'name': u'_internal'}, {u'name': u'monasca'}]
<type 'list'>
'''
db_exist = False
for current_dbname in database:
item = current_dbname
if dbname in (item[u'name']):
print("Database: %s is exist, switch database to %s") % (dbname, dbname)
db_exist = True
client.switch_database(dbname)
print("DB connected")
return client
if not db_exist:
print("DB is not exist, trying to create database.....")
client.create_database(dbname)
print("DB %s created, trying to switch database") % dbname
client.switch_database(dbname)
return client
except influxdb.client.InfluxDBClientError as e:
raise Exception(str(e))
def create_influxdb_database(self):
db = "ns_" + str(self.id)
client = InfluxDBClient(**INFLUXDB['default'])
print("Create database: " + db)
client.create_database(db)
print("Add grants")
client.grant_privilege("all", db, self.operator.name)
def send_to_influx(self):
if not self.config['hostname']:
self.log.error("No Influx server configured, please set one using: "
"ceph influx config-set hostname <hostname>")
return
# If influx server has authentication turned off then
# missing username/password is valid.
self.log.debug("Sending data to Influx host: %s",
self.config['hostname'])
client = InfluxDBClient(self.config['hostname'], self.config['port'],
self.config['username'],
self.config['password'],
self.config['database'],
self.config['ssl'],
self.config['verify_ssl'])
# using influx client get_list_database requires admin privs,
# instead we'll catch the not found exception and inform the user if
# db can not be created
try:
client.write_points(self.get_df_stats(), 'ms')
client.write_points(self.get_daemon_stats(), 'ms')
except InfluxDBClientError as e:
if e.code == 404:
self.log.info("Database '%s' not found, trying to create "
"(requires admin privs). You can also create "
"manually and grant write privs to user "
"'%s'", self.config['database'],
self.config['username'])
client.create_database(self.config['database'])
else:
raise
def send_to_influx(self):
if not self.config['hostname']:
self.log.error("No Influx server configured, please set one using: "
"ceph influx config-set hostname <hostname>")
self.set_health_checks({
'MGR_INFLUX_NO_SERVER': {
'severity': 'warning',
'summary': 'No InfluxDB server configured',
'detail': ['Configuration option hostname not set']
}
})
return
# If influx server has authentication turned off then
# missing username/password is valid.
self.log.debug("Sending data to Influx host: %s",
self.config['hostname'])
client = InfluxDBClient(self.config['hostname'], self.config['port'],
self.config['username'],
self.config['password'],
self.config['database'],
self.config['ssl'],
self.config['verify_ssl'])
# using influx client get_list_database requires admin privs,
# instead we'll catch the not found exception and inform the user if
# db can not be created
try:
client.write_points(self.get_df_stats(), 'ms')
client.write_points(self.get_daemon_stats(), 'ms')
self.set_health_checks(dict())
except ConnectionError as e:
self.log.exception("Failed to connect to Influx host %s:%d",
self.config['hostname'], self.config['port'])
self.set_health_checks({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB server at %s:%d'
' due to an connection error'
% (self.config['hostname'], self.config['port']),
'detail': [str(e)]
}
})
except InfluxDBClientError as e:
if e.code == 404:
self.log.info("Database '%s' not found, trying to create "
"(requires admin privs). You can also create "
"manually and grant write privs to user "
"'%s'", self.config['database'],
self.config['username'])
client.create_database(self.config['database'])
else:
self.set_health_checks({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB',
'detail': [str(e)]
}
})
raise
def main():
parser = argparse.ArgumentParser(description='whisper file to influxDB migration script')
parser.add_argument('path', help='path to whispers')
parser.add_argument('-host', default=DEFAULT_HOST, metavar="host", help="influxDB host")
parser.add_argument('-port', default=DEFAULT_PORT, metavar="port", help="influxDB port")
parser.add_argument('-user', default=DEFAULT_USER, metavar="user", help="influxDB user")
parser.add_argument('-password', default=DEFAULT_PASSWORD, metavar="password", help="influxDB password")
parser.add_argument('-db', default=DEFAULT_DB, metavar="db", help="influxDB db!")
args = parser.parse_args()
client = InfluxDBClient(args.host, args.port, args.user, args.password, args.db)
try: client.create_database(args.db)
except: pass
for whisper_file in search(args.path):
data = lame_whisper_read(whisper_file)
value = whisper_file.split('/')[-1].split('.')[0]
time_series = '.'.join(get_path_list(whisper_file)[:-1])
for key in data.iterkeys():
time = float(key)
#value = whisper_file.split('/')[-1].split('.')[0]
#time_series = whisper_file.replace('//','/').split('/')[-2].split('/')[-1]
#print time_series, value, time, data[key]
#time_info, values = whisper_read(whisper_file)
client.write_points(
[{
"name":time_series,
"columns":["time",value],
"points":[[time,data[key]]]
}])
def detect(runs, host, port):
results = []
# ZEUS malicious traffic detection (by looking at Domain)
# Cryptowall malicious POST request detection (by looking at URL)
# Exploit Kit malicious GET request detection (by looking at URL)
# Ponmuocup periodic malicious GET request detection (by looking at URL)
# connect to current traffic DB:
client = InfluxDBClient(host, port, database="Traffic_{}".format(runs))
# query
zeus = client.query("select * from http where Domain =~ /[a-z0-9]{32,48}.(info|biz|ru|com|org|net)/")
crypto_exploit = client.query(
"select * from http where (URL =~ /.\.php.*?./ and action = 'POST') or (URL =~ /^[0-9][0-9]\.[0-9][0-9]\.[0-9][0-9][0-9]\.[0-9][0-9][0-9]\/^?/ and action = 'GET')"
)
ponmo = client.query("SELECT * FROM http WHERE URL =~ /\/complete.search/ and action = 'GET'")
# analyze
results = predict_mal(zeus, crypto_exploit, ponmo)
# write analysis results to DB
client = InfluxDBClient(host, port, database="Detect_{}".format(runs))
client.create_database("Detect_{}".format(runs))
client.write_points(results)
def add_measurement_batch(meas_list, filename):
client1 = InfluxDBClient('localhost', 8086, 'root', 'root', DATABASE_NAME)
if DATABASE_NAME not in client.get_list_database():
client1.create_database(DATABASE_NAME)
client1.switch_user(DB_USER, DB_USER_PASSWORD)
client1.write_points(meas_list, batch_size=len(meas_list),
time_precision='ms', protocol='line')
class SkynetInflux:
def __init__(self):
# TODO: make this configurable.
# connect to the influx server
self.client = InfluxDBClient('localhost', 8086, 'root', 'root', 'skynet')
self.client.create_database('skynet', if_not_exists=True)
def get_results_for_plot(self, query):
pts = self.client.query(query)
results = list(pts.get_points())
retval = []
if len(results) > 0:
keys = list(results[0].keys())
keys.remove('time')
f = keys[0]
for r in results:
t = rfc3339_to_timestamp(r['time'])
d = r[f]
retval.append([t,d])
return retval
def get_measurements(self):
series = self.client.get_list_series()
measurements = []
for s in series:
print(s)
for t in s["tags"]:
try:
root = ".".join([t["board"], t["name"], s["name"]])
query = "SELECT * FROM %s WHERE \"board\"='%s' AND \"name\"='%s' LIMIT 1" % (s["name"], t["board"], t["name"])
print(query)
rs = self.client.query(query)
qr = list(rs.get_points())[0]
for k in qr.keys():
if k not in ['time', 'name', 'board', 'rtr', 'origin']:
measurements.append(root + "." + k)
except Exception as e:
print(e)
return sorted(measurements)
def get_points(measurement, start, stop, limit):
board, name, packet, data = measurement.split('.')
q = "SELECT %s FROM %s WHERE board='%s' AND name='%s' LIMIT %i"
class InfluxDB(object):
def __init__(self, db_host, db_port, db_name):
self._log = logging.getLogger(".".join([__name__, self.__class__.__name__]))
self._db_host = db_host
self._db_port = db_port
self._db_name = db_name
self._influx_client = None
self._connected = False
def connect(self):
self._log.info("Opening connection to influxDB at {:s}:{:d}".format(self._db_host, self._db_port))
try:
self._influx_client = InfluxDBClient(host=self._db_host, port=self._db_port)
existing_dbs = self._influx_client.get_list_database()
db_exists = False
for db in existing_dbs:
if db['name'] == self._db_name:
db_exists = True
break
if db_exists:
self._log.info("{} database exists already".format(self._db_name))
else:
self._log.info("Creating {} database".format(self._db_name))
self._influx_client.create_database(self._db_name)
self._influx_client.switch_database(self._db_name)
self._connected = True
except requests.ConnectionError:
self._log.info("Unable to connect to {} database".format(self._db_name))
def get_status(self):
status = {
"address": self._db_host,
"port": self._db_port,
"name": self._db_name,
"connected": self._connected
}
return status
def log_point(self, time, measurement, data):
if self._connected:
point = {
"measurement": measurement,
"time": time,
"fields": {}
}
for item in data:
point["fields"][item] = data[item]
self._influx_client.write_points([point])
def get_client(opts):
client = InfluxDBClient(opts.HOST, opts.PORT,
opts.user, opts.password,
opts.dbname)
try:
client.create_database(opts.dbname)
except InfluxDBClientError:
print >> sys.stderr, "debug: database %s exists. Continuing" % opts.dbname
return client
def test_ssh_poller(self):
""" Test real SSH connection to the server
"""
# Spawn mock SSH server process
p = Process(target=mock_cisco)
p.start()
sleep(1)
task = {
'hostname': 'localhost',
'username': '******',
'password': '******',
'port': 9999,
'device_type': 'cisco_nxos',
'parser_mode': 'fsm',
'precommands': '',
'interval': 0,
'commands': ['show interface:intf_name'],
}
poller = sshpoller.SSH_Poller(task)
# Connect to the SSH and send the command
poller.connect()
poller.send_commands()
# Generate a random name for our database
poller.db_name = 'testsshpoller_%s' % ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
# Create client and connect to the database
client = InfluxDBClient(host='127.0.0.1', port=poller.db_port, username=poller.db_user, password=poller.db_password)
client.create_database(poller.db_name)
# Write data to InfluxDB
poller.output_influxdb()
# Query database
query_results = client.query('select * from "show interface"', database=poller.db_name)
# Fetch expected results
expected_results = json.loads(open(os.path.join('mockssh', 'cisco_show_interface_influx.json'), 'r').read())
# Remove timestamps (the ones from the fixtures won't match the one we just wrote to the DB)
for i in expected_results['series'][0]['values']:
i[0] = ''
for i in query_results.raw['series'][0]['values']:
i[0] = ''
# Clean up
client.drop_database(poller.db_name)
# Kill server process
p.terminate()
self.assertEqual(query_results.raw, expected_results)
def main(host='localhost', port=8086, nb_day=15):
nb_day = 15 # number of day to generate time series
timeinterval_min = 5 # create an event every x minutes
total_minutes = 1440 * nb_day
total_records = int(total_minutes / timeinterval_min)
now = datetime.datetime.today()
metric = "server_data.cpu_idle"
series = []
for i in range(0, total_records):
past_date = now - datetime.timedelta(minutes=i * timeinterval_min)
value = random.randint(0, 200)
hostName = "server-%d" % random.randint(1, 5)
# pointValues = [int(past_date.strftime('%s')), value, hostName]
pointValues = {
"time": int(past_date.strftime('%s')),
"measurement": metric,
'fields': {
'value': value,
},
'tags': {
"hostName": hostName,
},
}
series.append(pointValues)
print(series)
client = InfluxDBClient(host, port, USER, PASSWORD, DBNAME)
print("Create database: " + DBNAME)
try:
client.create_database(DBNAME)
except InfluxDBClientError:
# Drop and create
client.drop_database(DBNAME)
client.create_database(DBNAME)
print("Create a retention policy")
retention_policy = 'server_data'
client.create_retention_policy(retention_policy, '3d', 3, default=True)
print("Write points #: {0}".format(total_records))
client.write_points(series, retention_policy=retention_policy)
time.sleep(2)
query = "SELECT MEAN(value) FROM {} WHERE time > now() - 10d GROUP BY time(500m)".format(metric)
result = client.query(query, database=DBNAME)
print(result)
print("Result: {0}".format(result))
print("Drop database: {}".format(DBNAME))
client.drop_database(DBNAME)
def main(host='localhost', port=8086):
user = '******'
password = '******'
dbname = 'example'
dbuser = '******'
dbuser_password = '******'
query = 'select column_one from foo;'
json_body = [{
"points": [
["1", 1, 1.0],
["2", 2, 2.0]
],
"name": "foo",
"columns": ["column_one", "column_two", "column_three"]
}]
client = InfluxDBClient(host, port, user, password, dbname)
print("Create database: " + dbname)
client.create_database(dbname)
dbusers = client.get_database_users()
print("Get list of database users: {0}".format(dbusers))
print("Add database user: "******"Make user a database admin")
client.set_database_admin(dbuser)
print("Remove admin privilege from user")
client.unset_database_admin(dbuser)
dbusers = client.get_database_users()
print("Get list of database users again: {0}".format(dbusers))
print("Switch user: "******"Write points: {0}".format(json_body))
client.write_points(json_body)
print("Queying data: " + query)
result = client.query(query)
print("Result: {0}".format(result))
print("Switch user: "******"Delete database: " + dbname)
client.delete_database(dbname)
class InfluxDBConnection:
def __init__(self, server="localhost", port=8086, user='******', password='******', database='skynet_lite'):
print "Connecting to Influx: %s@%s:%s/%s" % (user, server, port, database)
self.client = InfluxDBClient(server, port, user, password, database)
dbs = [x['name'] for x in self.client.get_list_database()]
if database not in dbs:
self.client.create_database(database)
def write_points(self, points):
self.client.write_points(points)
def main(host='localhost', port=8086):
user = '******'
password = '******'
dbname = 'Airfare'
json_body = get_airline_data()
client = InfluxDBClient(host, port, user, password, dbname)
client.delete_database(dbname)
print("Create database: " + dbname)
client.create_database(dbname)
print("Writing data")
client.write_points(json_body)
def writestream(count,array,M):
runs,host,port = system.givevars()
t0 = time.time()
client = InfluxDBClient(host.rstrip(), port.rstrip(), database='{"Traffic"_{}_{}'.format(runs,M))
client.create_database('{"Traffic"_{}_{}'.format(runs,M))
client.write_points(array,time_precision="ms")
t1 = time.time()
influx = t1 - t0
rate = float(count) / influx
f = open("log.txt", "wb")
f.write("SUCCESS. TIME INFLUX: {} WRITE SPEED PER REQUEST: {} DB: Traffic_{}_{}\n".format(influx, rate,runs,M))
f.close()
class TestResponseParser(unittest.TestCase):
def setUp(self):
self.client = InfluxDBClient('127.0.0.1', 8086, 'root', 'root')
self.database_name = 'test_%s' % (time.time())
self.client.create_database(self.database_name)
self.client.switch_db(self.database_name)
now = datetime.utcnow()
then = now - timedelta(hours=4)
self.points = geneate_points(then, now, timedelta(seconds=10), 10)
test_data = [{
"name": "test",
"columns": ["time", "key"],
"points": self.points
}]
self.client.write_points(data=test_data, batch_size=2000)
def tearDown(self):
self.client.delete_database(self.database_name)
def test_simple(self):
query = InfluxQuery.for_series('test').limit(None)
client = INDBClient(conn=self.client)
resp = client.result_for_query(query)
series = resp.get('test')
self.assertEqual(len(series), len(self.points))
query = query.limit(10)
resp = client.result_for_query(query)
series = resp.get('test')
assert len(series) == 10
query = InfluxQuery.for_series('test').columns(InfluxQuery.count('key')).limit(None)
resp = client.result_for_query(query)
series = resp.get('test')
self.assertEqual(series[0].count, len(self.points))
def test_groups(self):
q = InfluxQuery.for_series('test').columns(InfluxQuery.count('key'))
q = q.limit(None)
q = q.group_by(InfluxQuery.time('1h'))
client = INDBClient(conn=self.client)
resp = client.result_for_query(q)
series = resp.get('test')
assert sum(map(lambda x: x.count, series)) == len(self.points)
def start(argv):
"""
Instantiate an InfluxDBClient. The expected inputs are the host/address and
port of the InfluxDB and the name of the database to use. If the database
does not exist then it will be created. If the fourth arg is "auth" then it
will prompt the user for the InfluxDB's username and password.
"""
influxdb_host = argv[0]
influxdb_port = int(argv[1])
influxdb_name = argv[2]
if len(argv) > 3:
if argv[3] == "auth":
influxdb_username = raw_input("InfluxDB username: "******"Password: "******"Invalid args provided to %s: %s "\
"(expected: 'auth', got: '%s')" % (__name__, str(argv),
argv[3])
sys.exit(1)
else:
influxdb_username = "******"
influxdb_password = "******"
LOG.info("Connecting to: %s@%s:%d database:%s.",
influxdb_username, influxdb_host, influxdb_port, influxdb_name)
global g_client
g_client = InfluxDBClient(host=influxdb_host, port=influxdb_port,
database=influxdb_name,
username=influxdb_username,
password=influxdb_password)
create_database = True
try:
databases = g_client.get_list_database()
except (requests.exceptions.ConnectionError, InfluxDBClientError) as exc:
print >> sys.stderr, "Failed to connect to InfluxDB server at %s:%s "\
"database: %s.\nERROR: %s" % (influxdb_host,
str(influxdb_port), influxdb_name, str(exc))
sys.exit(1)
for database in databases:
if database["name"] == influxdb_name:
create_database = False
break
if create_database is True:
LOG.info("Creating database: %s.", influxdb_name)
g_client.create_database(influxdb_name)
class InfluxBackend(BaseBackend):
"""
InfluxDB backend.
:param database: name of the InfluxDB database.
:param client: client instance of InfluxDBClient class.
:param connection_url: InfluxDB connection url (influxdb://username:password@localhost:8086/databasename).
:param host: server host.
:param port: server port.
:param username: auth username.
:param password: auth password.
:param timeout: InfluxDB connection timeout (seconds).
"""
settings_namespace = 'INFLUX'
def __init__(self, database, client=None, connection_url=None, host=None, port=None, username=None, password=None,
timeout=0.3):
if not InfluxDBClient:
raise ImproperlyConfigured('You need to install the influxdb library to use the InfluxDB backend.')
if client:
if not isinstance(client, InfluxDBClient):
raise ImproperlyConfigured('"client" parameter is not an instance of InfluxDBClient client.')
self.client = client
elif connection_url:
self.client = InfluxDBClient.from_DSN(connection_url, timeout=timeout)
else:
self.client = InfluxDBClient(host=host, port=port, username=username, password=password,
database=database, timeout=timeout)
self.client.create_database(database)
def _get_payload(self, name, value, metric, tags, id_):
if tags:
tags['host'] = self._get_host_name()
else:
tags = {'host': self._get_host_name()}
if isinstance(value, dict):
fields = value
fields['name'] = name
else:
fields = {'name': name, 'value': value}
return [{'measurement': metric, 'time': datetime.utcnow(), 'tags': tags, 'fields': fields}]
def report(self, name, metric, value, tags, id_):
try:
payload = self._get_payload(name, value, metric, tags, id_)
return self.client.write_points(payload)
except Exception as e:
logger = logging.getLogger(__name__)
logger.exception(e)
class IdbConnection(object):
def __init__(self, user, pswd, host='localhost', port=8086, db='static'):
self.host = host
self.port = int(port)
self.client = InfluxDBClient(self.host, self.port, user, pswd, db)
self.client.create_database(db)
self.client.switch_database(db)
def table(self, name, tags=None):
"""
Return Table object, tags is list of columns that should be indexed
"""
return Table(self.client, name, tags)
def check_db_status():
# if the db is not found, then try to create it
try:
dbclient = InfluxDBClient(db_server, db_port, db_admin, db_admin_password)
dblist = dbclient.get_list_database()
db_found = False
for db in dblist:
if db['name'] == db_name:
db_found = True
if not(db_found):
logger.info('Database <%s> not found, trying to create it', db_name)
dbclient.create_database(db_name)
return True
except Exception as e:
logger.error('Error querying open-nti database: %s', e)
return False
def init_influx():
""" Initializes influx database. """
print("Connecting to Influx...")
influx_client = InfluxDBClient(
host=config.INFLUX_HOST,
port=config.INFLUX_PORT,
database=config.INFLUX_DB_NAME,
)
print("Connected to Influx!")
print("Creating influx database...")
influx_client.create_database(config.INFLUX_DB_NAME)
influx_client.create_retention_policy("one_week", "1w", 1, "listenbrainz")
print("Done!")
class TestInfluxDBClient(unittest.TestCase):
def setUp(self):
# By default, raise exceptions on warnings
warnings.simplefilter('error', FutureWarning)
self.cli = InfluxDBClient('localhost', 8086, 'username', 'password')
self.dummy_points = [{
"name": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"timestamp": "2009-11-10T23:00:00Z",
"fields": {
"value": 0.64
}
}]
def test_scheme(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
assert cli._baseurl == 'http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_write_points_batch(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 200, self.dummy_points):
assert cli.write_points(data=self.dummy_points,
batch_size=2) is True
def test_write_points_udp(self):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient('localhost',
8086,
'root',
'root',
'test',
use_udp=True,
udp_port=port)
cli.write_points(self.dummy_points)
received_data, addr = s.recvfrom(1024)
self.assertDictEqual({
"points": self.dummy_points,
"database": "test"
}, json.loads(received_data.decode(), strict=True))
def test_write_bad_precision_udp(self):
cli = InfluxDBClient('localhost',
8086,
'root',
'root',
'test',
use_udp=True,
udp_port=4444)
with self.assertRaisesRegexp(
Exception,
"InfluxDB only supports seconds precision for udp writes"):
cli.write_points(self.dummy_points, time_precision='ms')
@raises(Exception)
def test_write_points_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points([])
def test_write_points_with_precision(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST, "http://localhost:8086/write")
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points, time_precision='n')
self.assertDictEqual(
{
'points': self.dummy_points,
'database': 'db',
'precision': 'n',
}, json.loads(m.last_request.body))
def test_write_points_bad_precision(self):
cli = InfluxDBClient()
with self.assertRaisesRegexp(
Exception, "Invalid time precision is given. "
"\(use 'n', 'u', 'ms', 's', 'm' or 'h'\)"):
cli.write_points(self.dummy_points, time_precision='g')
@raises(Exception)
def test_write_points_with_precision_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points_with_precision([])
def test_query(self):
example_response = \
'{"results": [{"series": [{"name": "sdfsdfsdf", ' \
'"columns": ["time", "value"], "values": ' \
'[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": ' \
'[{"name": "cpu_load_short", "columns": ["time", "value"], ' \
'"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_query_chunked(self):
cli = InfluxDBClient(database='db')
example_object = {
'points': [[1415206250119, 40001, 667], [1415206244555, 30001, 7],
[1415206228241, 20001,
788], [1415206212980, 10001, 555],
[1415197271586, 10001, 23]],
'name':
'foo',
'columns': ['time', 'sequence_number', 'val']
}
example_response = \
json.dumps(example_object) + json.dumps(example_object)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/db/db/series",
text=example_response)
self.assertListEqual(cli.query('select * from foo', chunked=True),
[example_object, example_object])
@raises(Exception)
def test_query_fail(self):
with _mocked_session(self.cli, 'get', 401):
self.cli.query('select column_one from foo;')
def test_create_database(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.create_database('new_db')
self.assertEqual(m.last_request.qs['q'][0],
'create database new_db')
@raises(Exception)
def test_create_database_fails(self):
with _mocked_session(self.cli, 'post', 401):
self.cli.create_database('new_db')
def test_drop_database(self):
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.drop_database('new_db')
self.assertEqual(m.last_request.qs['q'][0], 'drop database new_db')
@raises(Exception)
def test_drop_database_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'delete', 401):
cli.drop_database('old_db')
def test_get_list_database(self):
data = {
'results': [{
'series': [{
'name': 'databases',
'columns': ['name'],
'values': [['mydb'], ['myotherdb']]
}]
}]
}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(self.cli.get_list_database(),
['mydb', 'myotherdb'])
@raises(Exception)
def test_get_list_database_fails(self):
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_database()
def test_get_list_series(self):
example_response = \
'{"results": [{"series": [{"values": [["fsfdsdf", "24h0m0s", 2]],'\
' "columns": ["name", "duration", "replicaN"]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://*****:*****@mock.patch('requests.Session.request')
def test_request_retry(self, mock_request):
"""Tests that two connection errors will be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 3:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points)
@mock.patch('requests.Session.request')
def test_request_retry_raises(self, mock_request):
"""Tests that three connection errors will not be handled"""
class CustomMock(object):
i = 0
def connection_error(self, *args, **kwargs):
self.i += 1
if self.i < 4:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
with self.assertRaises(requests.exceptions.ConnectionError):
cli.write_points(self.dummy_points)
class TestInfluxDBClient(unittest.TestCase):
"""Set up the TestInfluxDBClient object."""
def setUp(self):
"""Initialize an instance of TestInfluxDBClient object."""
# By default, raise exceptions on warnings
warnings.simplefilter('error', FutureWarning)
self.cli = InfluxDBClient('localhost', 8086, 'username', 'password')
self.dummy_points = [{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00.123456Z",
"fields": {
"value": 0.64
}
}]
self.dsn_string = 'influxdb://*****:*****@my.host.fr:1886/db'
def test_scheme(self):
"""Set up the test schema for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
self.assertEqual('http://host:8086', cli._baseurl)
cli = InfluxDBClient('host',
8086,
'username',
'password',
'database',
ssl=True)
self.assertEqual('https://host:8086', cli._baseurl)
def test_dsn(self):
"""Set up the test datasource name for TestInfluxDBClient object."""
cli = InfluxDBClient.from_dsn('influxdb://192.168.0.1:1886')
self.assertEqual('http://192.168.0.1:1886', cli._baseurl)
cli = InfluxDBClient.from_dsn(self.dsn_string)
self.assertEqual('http://my.host.fr:1886', cli._baseurl)
self.assertEqual('uSr', cli._username)
self.assertEqual('pWd', cli._password)
self.assertEqual('db', cli._database)
self.assertFalse(cli._use_udp)
cli = InfluxDBClient.from_dsn('udp+' + self.dsn_string)
self.assertTrue(cli._use_udp)
cli = InfluxDBClient.from_dsn('https+' + self.dsn_string)
self.assertEqual('https://my.host.fr:1886', cli._baseurl)
cli = InfluxDBClient.from_dsn('https+' + self.dsn_string,
**{'ssl': False})
self.assertEqual('http://my.host.fr:1886', cli._baseurl)
def test_switch_database(self):
"""Test switch database in TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
cli.switch_database('another_database')
self.assertEqual('another_database', cli._database)
def test_switch_user(self):
"""Test switch user in TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'database')
cli.switch_user('another_username', 'another_password')
self.assertEqual('another_username', cli._username)
self.assertEqual('another_password', cli._password)
def test_write(self):
"""Test write in TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write({
"database":
"mydb",
"retentionPolicy":
"mypolicy",
"points": [{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"value": 0.64
}
}]
})
self.assertEqual(
m.last_request.body,
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000000000000\n",
)
def test_write_points(self):
"""Test write points for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points, )
self.assertEqual(
'cpu_load_short,host=server01,region=us-west '
'value=0.64 1257894000123456000\n',
m.last_request.body.decode('utf-8'),
)
def test_write_points_toplevel_attributes(self):
"""Test write points attrs for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points,
database='testdb',
tags={"tag": "hello"},
retention_policy="somepolicy")
self.assertEqual(
'cpu_load_short,host=server01,region=us-west,tag=hello '
'value=0.64 1257894000123456000\n',
m.last_request.body.decode('utf-8'),
)
def test_write_points_batch(self):
"""Test write points batch for TestInfluxDBClient object."""
dummy_points = [{
"measurement": "cpu_usage",
"tags": {
"unit": "percent"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"value": 12.34
}
}, {
"measurement": "network",
"tags": {
"direction": "in"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"value": 123.00
}
}, {
"measurement": "network",
"tags": {
"direction": "out"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"value": 12.00
}
}]
expected_last_body = (
"network,direction=out,host=server01,region=us-west "
"value=12.0 1257894000000000000\n")
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write_points(points=dummy_points,
database='db',
tags={
"host": "server01",
"region": "us-west"
},
batch_size=2)
self.assertEqual(m.call_count, 2)
self.assertEqual(expected_last_body,
m.last_request.body.decode('utf-8'))
def test_write_points_udp(self):
"""Test write points UDP for TestInfluxDBClient object."""
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient('localhost',
8086,
'root',
'root',
'test',
use_udp=True,
udp_port=port)
cli.write_points(self.dummy_points)
received_data, addr = s.recvfrom(1024)
self.assertEqual(
'cpu_load_short,host=server01,region=us-west '
'value=0.64 1257894000123456000\n', received_data.decode())
@raises(Exception)
def test_write_points_fails(self):
"""Test write points fail for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points([])
def test_write_points_with_precision(self):
"""Test write points with precision for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.POST,
"http://localhost:8086/write",
status_code=204)
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points, time_precision='n')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456000\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='u')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='ms')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='s')
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000\n",
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='m')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 20964900\n',
m.last_request.body,
)
cli.write_points(self.dummy_points, time_precision='h')
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 349415\n',
m.last_request.body,
)
def test_write_points_with_precision_udp(self):
"""Test write points with precision for TestInfluxDBClient object."""
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = random.randint(4000, 8000)
s.bind(('0.0.0.0', port))
cli = InfluxDBClient('localhost',
8086,
'root',
'root',
'test',
use_udp=True,
udp_port=port)
cli.write_points(self.dummy_points, time_precision='n')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456000\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='u')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123456\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='ms')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 1257894000123\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='s')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b"cpu_load_short,host=server01,region=us-west "
b"value=0.64 1257894000\n",
received_data,
)
cli.write_points(self.dummy_points, time_precision='m')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 20964900\n',
received_data,
)
cli.write_points(self.dummy_points, time_precision='h')
received_data, addr = s.recvfrom(1024)
self.assertEqual(
b'cpu_load_short,host=server01,region=us-west '
b'value=0.64 349415\n',
received_data,
)
def test_write_points_bad_precision(self):
"""Test write points w/bad precision TestInfluxDBClient object."""
cli = InfluxDBClient()
with self.assertRaisesRegexp(
Exception, "Invalid time precision is given. "
"\(use 'n', 'u', 'ms', 's', 'm' or 'h'\)"):
cli.write_points(self.dummy_points, time_precision='g')
@raises(Exception)
def test_write_points_with_precision_fails(self):
"""Test write points w/precision fail for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password', 'db')
with _mocked_session(cli, 'post', 500):
cli.write_points_with_precision([])
def test_query(self):
"""Test query method for TestInfluxDBClient object."""
example_response = (
'{"results": [{"series": [{"measurement": "sdfsdfsdf", '
'"columns": ["time", "value"], "values": '
'[["2009-11-10T23:00:00Z", 0.64]]}]}, {"series": '
'[{"measurement": "cpu_load_short", "columns": ["time", "value"], '
'"values": [["2009-11-10T23:00:00Z", 0.64]]}]}]}')
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://*****:*****@unittest.skip('Not implemented for 0.9')
def test_query_chunked(self):
"""Test chunked query for TestInfluxDBClient object."""
cli = InfluxDBClient(database='db')
example_object = {
'points': [[1415206250119, 40001, 667], [1415206244555, 30001, 7],
[1415206228241, 20001,
788], [1415206212980, 10001, 555],
[1415197271586, 10001, 23]],
'measurement':
'foo',
'columns': ['time', 'sequence_number', 'val']
}
example_response = \
json.dumps(example_object) + json.dumps(example_object)
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/db/db/series",
text=example_response)
self.assertListEqual(cli.query('select * from foo', chunked=True),
[example_object, example_object])
@raises(Exception)
def test_query_fail(self):
"""Test query failed for TestInfluxDBClient object."""
with _mocked_session(self.cli, 'get', 401):
self.cli.query('select column_one from foo;')
def test_ping(self):
"""Test ping querying InfluxDB version."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/ping",
status_code=204,
headers={'X-Influxdb-Version': '1.2.3'})
version = self.cli.ping()
self.assertEqual(version, '1.2.3')
def test_create_database(self):
"""Test create database for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.create_database('new_db')
self.assertEqual(m.last_request.qs['q'][0],
'create database "new_db"')
def test_create_numeric_named_database(self):
"""Test create db w/numeric name for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.create_database('123')
self.assertEqual(m.last_request.qs['q'][0],
'create database "123"')
@raises(Exception)
def test_create_database_fails(self):
"""Test create database fail for TestInfluxDBClient object."""
with _mocked_session(self.cli, 'post', 401):
self.cli.create_database('new_db')
def test_drop_database(self):
"""Test drop database for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.drop_database('new_db')
self.assertEqual(m.last_request.qs['q'][0],
'drop database "new_db"')
def test_drop_measurement(self):
"""Test drop measurement for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.drop_measurement('new_measurement')
self.assertEqual(m.last_request.qs['q'][0],
'drop measurement "new_measurement"')
def test_drop_numeric_named_database(self):
"""Test drop numeric db for TestInfluxDBClient object."""
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text='{"results":[{}]}')
self.cli.drop_database('123')
self.assertEqual(m.last_request.qs['q'][0], 'drop database "123"')
def test_get_list_database(self):
"""Test get list of databases for TestInfluxDBClient object."""
data = {
'results': [{
'series': [{
'name': 'databases',
'values': [['new_db_1'], ['new_db_2']],
'columns': ['name']
}]
}]
}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(self.cli.get_list_database(),
[{
'name': 'new_db_1'
}, {
'name': 'new_db_2'
}])
@raises(Exception)
def test_get_list_database_fails(self):
"""Test get list of dbs fail for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_database()
def test_get_list_measurements(self):
"""Test get list of measurements for TestInfluxDBClient object."""
data = {
"results": [{
"series": [{
"name": "measurements",
"columns": ["name"],
"values": [["cpu"], ["disk"]]
}]
}]
}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(self.cli.get_list_measurements(),
[{
'name': 'cpu'
}, {
'name': 'disk'
}])
def test_create_retention_policy_default(self):
"""Test create default ret policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.create_retention_policy('somename',
'1d',
4,
default=True,
database='db')
self.assertEqual(
m.last_request.qs['q'][0],
'create retention policy "somename" on '
'"db" duration 1d replication 4 default')
def test_create_retention_policy(self):
"""Test create retention policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.create_retention_policy('somename',
'1d',
4,
database='db')
self.assertEqual(
m.last_request.qs['q'][0],
'create retention policy "somename" on '
'"db" duration 1d replication 4')
def test_alter_retention_policy(self):
"""Test alter retention policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
# Test alter duration
self.cli.alter_retention_policy('somename', 'db', duration='4d')
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" duration 4d')
# Test alter replication
self.cli.alter_retention_policy('somename', 'db', replication=4)
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" replication 4')
# Test alter default
self.cli.alter_retention_policy('somename', 'db', default=True)
self.assertEqual(
m.last_request.qs['q'][0],
'alter retention policy "somename" on "db" default')
@raises(Exception)
def test_alter_retention_policy_invalid(self):
"""Test invalid alter ret policy for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.alter_retention_policy('somename', 'db')
def test_drop_retention_policy(self):
"""Test drop retention policy for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.drop_retention_policy('somename', 'db')
self.assertEqual(m.last_request.qs['q'][0],
'drop retention policy "somename" on "db"')
@raises(Exception)
def test_drop_retention_policy_fails(self):
"""Test failed drop ret policy for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'delete', 401):
cli.drop_retention_policy('default', 'db')
def test_get_list_retention_policies(self):
"""Test get retention policies for TestInfluxDBClient object."""
example_response = \
'{"results": [{"series": [{"values": [["fsfdsdf", "24h0m0s", 2]],'\
' "columns": ["name", "duration", "replicaN"]}]}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://*****:*****@mock.patch('requests.Session.request')
def test_request_retry(self, mock_request):
"""Test that two connection errors will be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self):
self.i = 0
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < 3:
raise requests.exceptions.ConnectionError
r = requests.Response()
r.status_code = 204
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
cli.write_points(self.dummy_points)
@mock.patch('requests.Session.request')
def test_request_retry_raises(self, mock_request):
"""Test that three requests errors will not be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self):
self.i = 0
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < 4:
raise requests.exceptions.HTTPError
else:
r = requests.Response()
r.status_code = 200
return r
mock_request.side_effect = CustomMock().connection_error
cli = InfluxDBClient(database='db')
with self.assertRaises(requests.exceptions.HTTPError):
cli.write_points(self.dummy_points)
@mock.patch('requests.Session.request')
def test_random_request_retry(self, mock_request):
"""Test that a random number of connection errors will be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self, retries):
self.i = 0
self.retries = retries
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < self.retries:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 204
return r
retries = random.randint(1, 5)
mock_request.side_effect = CustomMock(retries).connection_error
cli = InfluxDBClient(database='db', retries=retries)
cli.write_points(self.dummy_points)
@mock.patch('requests.Session.request')
def test_random_request_retry_raises(self, mock_request):
"""Test a random number of conn errors plus one will not be handled."""
class CustomMock(object):
"""Create custom mock object for test."""
def __init__(self, retries):
self.i = 0
self.retries = retries
def connection_error(self, *args, **kwargs):
"""Handle a connection error for the CustomMock object."""
self.i += 1
if self.i < self.retries + 1:
raise requests.exceptions.ConnectionError
else:
r = requests.Response()
r.status_code = 200
return r
retries = random.randint(1, 5)
mock_request.side_effect = CustomMock(retries).connection_error
cli = InfluxDBClient(database='db', retries=retries)
with self.assertRaises(requests.exceptions.ConnectionError):
cli.write_points(self.dummy_points)
def test_get_list_users(self):
"""Test get users for TestInfluxDBClient object."""
example_response = (
'{"results":[{"series":[{"columns":["user","admin"],'
'"values":[["test",false]]}]}]}')
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.assertListEqual(self.cli.get_list_users(), [{
'user': '******',
'admin': False
}])
def test_get_list_users_empty(self):
"""Test get empty userlist for TestInfluxDBClient object."""
example_response = (
'{"results":[{"series":[{"columns":["user","admin"]}]}]}')
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.assertListEqual(self.cli.get_list_users(), [])
def test_grant_admin_privileges(self):
"""Test grant admin privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.grant_admin_privileges('test')
self.assertEqual(m.last_request.qs['q'][0],
'grant all privileges to "test"')
@raises(Exception)
def test_grant_admin_privileges_invalid(self):
"""Test grant invalid admin privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.grant_admin_privileges('')
def test_revoke_admin_privileges(self):
"""Test revoke admin privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.revoke_admin_privileges('test')
self.assertEqual(m.last_request.qs['q'][0],
'revoke all privileges from "test"')
@raises(Exception)
def test_revoke_admin_privileges_invalid(self):
"""Test revoke invalid admin privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.revoke_admin_privileges('')
def test_grant_privilege(self):
"""Test grant privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.grant_privilege('read', 'testdb', 'test')
self.assertEqual(m.last_request.qs['q'][0],
'grant read on "testdb" to "test"')
@raises(Exception)
def test_grant_privilege_invalid(self):
"""Test grant invalid privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.grant_privilege('', 'testdb', 'test')
def test_revoke_privilege(self):
"""Test revoke privs for TestInfluxDBClient object."""
example_response = '{"results":[{}]}'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
self.cli.revoke_privilege('read', 'testdb', 'test')
self.assertEqual(m.last_request.qs['q'][0],
'revoke read on "testdb" from "test"')
@raises(Exception)
def test_revoke_privilege_invalid(self):
"""Test revoke invalid privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 400):
self.cli.revoke_privilege('', 'testdb', 'test')
def test_get_list_privileges(self):
"""Tst get list of privs for TestInfluxDBClient object."""
data = {
'results': [{
'series': [{
'columns': ['database', 'privilege'],
'values': [['db1', 'READ'], ['db2', 'ALL PRIVILEGES'],
['db3', 'NO PRIVILEGES']]
}]
}]
}
with _mocked_session(self.cli, 'get', 200, json.dumps(data)):
self.assertListEqual(self.cli.get_list_privileges('test'),
[{
'database': 'db1',
'privilege': 'READ'
}, {
'database': 'db2',
'privilege': 'ALL PRIVILEGES'
}, {
'database': 'db3',
'privilege': 'NO PRIVILEGES'
}])
@raises(Exception)
def test_get_list_privileges_fails(self):
"""Test failed get list of privs for TestInfluxDBClient object."""
cli = InfluxDBClient('host', 8086, 'username', 'password')
with _mocked_session(cli, 'get', 401):
cli.get_list_privileges('test')
def test_invalid_port_fails(self):
"""Test invalid port fail for TestInfluxDBClient object."""
with self.assertRaises(ValueError):
InfluxDBClient('host', '80/redir', 'username', 'password')
def test_chunked_response(self):
"""Test chunked reponse for TestInfluxDBClient object."""
example_response = \
u'{"results":[{"statement_id":0,"series":' \
'[{"name":"cpu","columns":["fieldKey","fieldType"],"values":' \
'[["value","integer"]]}],"partial":true}]}\n{"results":' \
'[{"statement_id":0,"series":[{"name":"iops","columns":' \
'["fieldKey","fieldType"],"values":[["value","integer"]]}],' \
'"partial":true}]}\n{"results":[{"statement_id":0,"series":' \
'[{"name":"load","columns":["fieldKey","fieldType"],"values":' \
'[["value","integer"]]}],"partial":true}]}\n{"results":' \
'[{"statement_id":0,"series":[{"name":"memory","columns":' \
'["fieldKey","fieldType"],"values":[["value","integer"]]}]}]}\n'
with requests_mock.Mocker() as m:
m.register_uri(requests_mock.GET,
"http://localhost:8086/query",
text=example_response)
response = self.cli.query('show series limit 4 offset 0',
chunked=True,
chunk_size=4)
self.assertTrue(len(response) == 4)
self.assertEqual(
response.__repr__(),
ResultSet({
'series': [{
'values': [['value', 'integer']],
'name': 'cpu',
'columns': ['fieldKey', 'fieldType']
}, {
'values': [['value', 'integer']],
'name': 'iops',
'columns': ['fieldKey', 'fieldType']
}, {
'values': [['value', 'integer']],
'name': 'load',
'columns': ['fieldKey', 'fieldType']
}, {
'values': [['value', 'integer']],
'name': 'memory',
'columns': ['fieldKey', 'fieldType']
}]
}).__repr__())
def create_influxdb(app_id,
database_name="asperathos",
img="influxdb",
namespace="default",
visualizer_port=8086,
timeout=60):
kube.config.load_kube_config(api.k8s_conf_path)
influx_pod_spec = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"name": "influxdb-%s" % app_id,
"labels": {
"app": "influxdb-%s" % app_id
}
},
"spec": {
"containers": [{
"name": "influxdb-master",
"image": img,
"env": [{
"name": "MASTER",
"value": str(True)
}],
"ports": [{
"containerPort": visualizer_port
}]
}]
}
}
influx_svc_spec = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": "influxdb-%s" % app_id,
"labels": {
"app": "influxdb-%s" % app_id
}
},
"spec": {
"ports": [{
"protocol": "TCP",
"port": visualizer_port,
"targetPort": visualizer_port
}],
"selector": {
"app": "influxdb-%s" % app_id
},
"type":
"NodePort"
}
}
CoreV1Api = kube.client.CoreV1Api()
node_port = None
# Gets the redis ip if the value is not explicitic in the config file
try:
redis_ip = api.redis_ip
except AttributeError:
redis_ip = api.get_node_cluster(api.k8s_conf_path)
try:
KUBEJOBS_LOG.log("Creating InfluxDB Pod...")
CoreV1Api.create_namespaced_pod(namespace=namespace,
body=influx_pod_spec)
KUBEJOBS_LOG.log("Creating InfluxDB Service...")
s = CoreV1Api.create_namespaced_service(namespace=namespace,
body=influx_svc_spec)
ready = False
attempts = timeout
while not ready:
read = CoreV1Api.read_namespaced_pod_status(name="influxdb-%s" %
app_id,
namespace=namespace)
node_port = s.spec.ports[0].node_port
if read.status.phase == "Running" and node_port is not None:
try:
# TODO change redis_ip to node_ip
client = InfluxDBClient(redis_ip, node_port, 'root',
'root', database_name)
client.create_database(database_name)
KUBEJOBS_LOG.log("InfluxDB is ready!!")
ready = True
except Exception:
KUBEJOBS_LOG.log("InfluxDB is not ready yet...")
else:
attempts -= 1
if attempts > 0:
time.sleep(1)
else:
raise Exception("InfluxDB cannot be started!"
"Time limite exceded...")
time.sleep(1)
influxdb_data = {"port": node_port, "name": database_name}
return influxdb_data
except kube.client.rest.ApiException as e:
KUBEJOBS_LOG.log(e)
},
'tags': []
} for key in ['download', 'upload', 'ping']]
db.write_points(res)
def run_threaded(job_func):
job_thread = Thread(target=job_func)
job_thread.start()
_LOGGER.info('Initialising ...')
global db
db = InfluxDBClient(host='influxdb',
port=8086,
username=_INFLUX_DB_USER,
password=_INFLUX_DB_PASSWORD,
database='speedtest')
db.create_database('speedtest')
db.create_retention_policy('forever', 'INF', 1, default=True)
db.switch_user("dbuser", "dbuser")
_LOGGER.info('Initialised')
job() # run immediately
schedule.every(_SCHEDULE_INTERVAL_MINS).minutes.do(run_threaded, job)
while 1:
schedule.run_pending()
time.sleep(1)
from influxdb import InfluxDBClient
import csv
import json
import random
client = InfluxDBClient('localhost', 8086, 'root', 'root',
'complete_data_result')
# client.drop_database('complete_data')
client.create_database('complete_data_result')
buffer_length = 4000
buffered_payload = []
def writeResults(row0, row1, row2, row3, label):
row_count = 0
global buffered_payload
for i in range(len(label)):
data = {}
data['measurement'] = "iot_fault_data_result"
data['time'] = row0[i]
fields = {}
fields['load'] = row1[i]
fields['rate'] = row2[i]
fields['gs'] = row3[i]
fields['label'] = label[i]
data['fields'] = fields
buffered_payload.append(data)
row_count = row_count + 1
if row_count == buffer_length:
from datetime import datetime
from influxdb import InfluxDBClient
client = InfluxDBClient(host='localhost',
port=8086,
username='******',
password='******',
timeout=3,
database='empower')
client.create_database('empower')
timestamp = datetime.utcnow()
# slices_rates : <id_slice>: <rate> (rate en MBits/s)
# lvap_slice :
# para cada slice generar un punto, con tag slice igual al id de la slice
# en campos poner los lvaps: <numero>: <lvap> donde numero no se puede repetir
points = [
{
"measurement": "slices_rates",
"tags": {},
"time": timestamp,
"fields": {
"1": 2.0,
"2": 5.0,
"3": 10.0,
"4": 15.0
}
},
{
def send_to_influx(self):
if not self.config['hostname']:
self.log.error(
"No Influx server configured, please set one using: "
"ceph influx config-set hostname <hostname>")
self.set_health_checks({
'MGR_INFLUX_NO_SERVER': {
'severity': 'warning',
'summary': 'No InfluxDB server configured',
'detail': ['Configuration option hostname not set']
}
})
return
# If influx server has authentication turned off then
# missing username/password is valid.
self.log.debug("Sending data to Influx host: %s",
self.config['hostname'])
client = InfluxDBClient(self.config['hostname'], self.config['port'],
self.config['username'],
self.config['password'],
self.config['database'], self.config['ssl'],
self.config['verify_ssl'])
# using influx client get_list_database requires admin privs,
# instead we'll catch the not found exception and inform the user if
# db can not be created
try:
df_stats, pools = self.get_df_stats()
client.write_points(df_stats, 'ms')
client.write_points(self.get_daemon_stats(), 'ms')
client.write_points(self.get_pg_summary(pools))
self.set_health_checks(dict())
except ConnectionError as e:
self.log.exception("Failed to connect to Influx host %s:%d",
self.config['hostname'], self.config['port'])
self.set_health_checks({
'MGR_INFLUX_SEND_FAILED': {
'severity':
'warning',
'summary':
'Failed to send data to InfluxDB server at %s:%d'
' due to an connection error' %
(self.config['hostname'], self.config['port']),
'detail': [str(e)]
}
})
except InfluxDBClientError as e:
if e.code == 404:
self.log.info(
"Database '%s' not found, trying to create "
"(requires admin privs). You can also create "
"manually and grant write privs to user "
"'%s'", self.config['database'], self.config['username'])
client.create_database(self.config['database'])
client.create_retention_policy(
name='8_weeks',
duration='8w',
replication='1',
default=True,
database=self.config['database'])
else:
self.set_health_checks({
'MGR_INFLUX_SEND_FAILED': {
'severity': 'warning',
'summary': 'Failed to send data to InfluxDB',
'detail': [str(e)]
}
})
raise
from influxdb import InfluxDBClient
# tsdb = InfluxDBClient('192.168.0.11', 8086, 'root', 'root', 'weather_data')
tsdb = InfluxDBClient('influxdb', 8086, 'root', 'root', 'weather_data')
tsdb.create_database('weather_data')
import asyncio
import asyncpg
async def init_datastores():
pass
# global pgdb
# pgdb = await asyncpg.create_pool(database='postgres',
# user='******', password='******')
loop = asyncio.get_event_loop()
loop.run_until_complete(init_datastores())
# async def run():
# conn = await asyncpg.connect(user='******', password='******',
# database='database', host='127.0.0.1')
# values = await conn.fetch('''SELECT * FROM mytable''')
# await conn.close()
def influx_database():
influx = InfluxDBClient()
influx.create_database('coinrat_test')
influx._database = 'coinrat_test'
yield influx
influx.drop_database('coinrat_test')
class InfluxDBListener:
"""
Events listener that writes locust events to the given influxdb connection
"""
def __init__(
self,
env: locust.env.Environment,
influxDbSettings: InfluxDBSettings
):
# flush related attributes
self.cache = []
self.stop_flag = False
self.interval_ms = influxDbSettings.interval_ms
# influxdb settings
try:
self.influxdb_client = InfluxDBClient(influxDbSettings.influx_host, influxDbSettings.influx_port, influxDbSettings.user, influxDbSettings.pwd, influxDbSettings.database)
self.influxdb_client.create_database(influxDbSettings.database)
except:
logging.exception('Could not connect to influxdb')
return
# determine if worker or master
self.node_id = 'local'
if '--master' in sys.argv:
self.node_id = 'master'
if '--worker' in sys.argv:
# TODO: Get real ID of slaves form locust somehow
self.node_id = 'worker'
# start background event to push data to influx
self.flush_worker = gevent.spawn(self.__flush_cached_points_worker)
self.test_start(0)
events = env.events
# requests
events.request_success.add_listener(self.request_success)
events.request_failure.add_listener(self.request_failure)
# events
events.test_stop.add_listener(self.test_stop)
events.user_error.add_listener(self.user_error)
events.spawning_complete.add_listener(self.spawning_complete)
events.quitting.add_listener(self.quitting)
# complete
atexit.register(self.quitting)
def request_success(self, request_type, name, response_time, response_length, **_kwargs) -> None:
self.__listen_for_requests_events(self.node_id, 'locust_requests', request_type, name, response_time, response_length, True, None)
def request_failure(self, request_type, name, response_time, response_length, exception, **_kwargs) -> None:
self.__listen_for_requests_events(self.node_id, 'locust_requests', request_type, name, response_time, response_length, False, exception)
def spawning_complete(self, user_count) -> None:
self.__register_event(self.node_id, user_count, 'spawning_complete')
return True
def test_start(self, user_count) -> None:
self.__register_event(self.node_id, 0, 'test_started')
def test_stop(self, user_count) -> None:
print('test started')
self.__register_event(self.node_id, 0, 'test_stopped')
def test_stop(self, user_count) -> None:
self.__register_event(self.node_id, 0, 'test_stopped')
def user_error(self, user_instance, exception, tb, **_kwargs) -> None:
self.__listen_for_locust_errors(self.node_id, user_instance, exception, tb)
def quitting(self, **_kwargs) -> None:
self.__register_event(self.node_id, 0, 'quitting')
self.last_flush_on_quitting()
def __register_event(self, node_id: str, user_count: int, event: str, **_kwargs) -> None:
"""
Persist locust event such as hatching started or stopped to influxdb.
Append user_count in case that it exists
:param node_id: The id of the node reporting the event.
:param event: The event name or description.
"""
time = datetime.utcnow()
tags = {
}
fields = {
'node_id': node_id,
'event': event,
'user_count': user_count
}
point = self.__make_data_point('locust_events', tags, fields, time)
self.cache.append(point)
def __listen_for_requests_events(self, node_id, measurement, request_type, name, response_time, response_length, success, exception) -> None:
"""
Persist request information to influxdb.
:param node_id: The id of the node reporting the event.
:param measurement: The measurement where to save this point.
:param success: Flag the info to as successful request or not
"""
time = datetime.utcnow()
tags = {
'node_id': node_id,
'request_type': request_type,
'name': name,
'success': success,
'exception': repr(exception),
}
if isinstance(exception, HTTPError):
tags['code'] = exception.response.status_code
fields = {
'response_time': response_time,
'response_length': response_length,
'counter': 1, # TODO: Review the need of this field
}
point = self.__make_data_point(measurement, tags, fields, time)
self.cache.append(point)
def __listen_for_locust_errors(self, node_id, user_instance, exception: Exception = None, tb = None) -> None:
"""
Persist locust errors to InfluxDB.
:param node_id: The id of the node reporting the error.
:return: None
"""
time = datetime.utcnow()
tags = {
'exception_tag': repr(exception)
}
fields = {
'node_id': node_id,
'user_instance': repr(user_instance),
'exception': repr(exception),
'traceback': "".join(traceback.format_tb(tb)),
}
point = self.__make_data_point('locust_exceptions', tags, fields, time)
self.cache.append(point)
def __flush_cached_points_worker(self) -> None:
"""
Background job that puts the points into the cache to be flushed according tot he interval defined.
:param influxdb_client:
:param interval:
:return: None
"""
log.info('Flush worker started.')
while not self.stop_flag:
self.__flush_points(self.influxdb_client)
gevent.sleep(self.interval_ms / 1000)
def __make_data_point(self, measurement: str, tags: dict, fields: dict, time: datetime) -> dict:
"""
Create a list with a single point to be saved to influxdb.
:param measurement: The measurement where to save this point.
:param tags: Dictionary of tags to be saved in the measurement.
:param fields: Dictionary of field to be saved to measurement.
:param time: The time os this point.
"""
return {"measurement": measurement, "tags": tags, "time": time, "fields": fields}
def last_flush_on_quitting(self):
self.stop_flag = True
self.flush_worker.join()
self.__flush_points(self.influxdb_client)
def __flush_points(self, influxdb_client: InfluxDBClient) -> None:
"""
Write the cached data points to influxdb
:param influxdb_client: An instance of InfluxDBClient
:return: None
"""
log.debug(f'Flushing points {len(self.cache)}')
to_be_flushed = self.cache
self.cache = []
success = influxdb_client.write_points(to_be_flushed)
if not success:
log.error('Failed to write points to influxdb.')
# If failed for any reason put back into the beginning of cache
self.cache.insert(0, to_be_flushed)
class InfluxdbUtils(object):
"""
influxdb operator utils.
"""
def __init__(self,
host=u'localhost',
port=8086,
username=u'root',
password=u'root',
dbname=u'vzhucloud'):
self.host = host
self.port = port
self.username = username
self.password = password
self.dbname = dbname
def connect(self):
"""
Connect to influxdb server.
"""
self.client = InfluxDBClient(host=self.host,
port=self.port,
username=self.username,
password=self.password,
timeout=30)
databases = self.client.get_list_database()
if self.dbname not in databases:
try:
self.client.create_database(self.dbname)
except InfluxDBClientError:
self.client.drop_database(self.dbname)
self.client.create_database(self.dbname)
self.client.switch_user(self.username, self.password)
self.client.switch_database(self.dbname)
def create_retention_policy(self,
name=u'30days',
duration='30d',
replication='1'):
"""
Create a retention policy for a database.
:param name (str) – the name of the new retention policy
:param duration (str) – the duration of the new retention policy.
Durations such as 1h, 90m, 12h, 7d, and 4w, are all supported
and mean 1 hour, 90 minutes, 12 hours, 7 day, and 4 weeks, respectively.
:param replication (str) – the replication of the retention policy
"""
ret_policies = self.client.get_list_retention_policies(self.dbname)
for policy in ret_policies:
if policy['name'] == name:
return
self.client.create_retention_policy(name, duration, replication,
self.dbname, True)
def write(self, data):
"""
Write time series data to influxdb.
:param data (json array) - as blew
[{
"measurement": "vm_monitor_data",
"tags": {
"host": "5e4643d3-fab0-4adb-8a13-a9d278e28686",
"region": "wuxi-test-compute144",
"object": "disk",
"metric": "ops_read",
"instance": "/dev/sda1"
},
"time": "2017-03-12T22:00:00Z",
"fields": {
"value": 89
}
}]
"""
if not self.client.write_points(data):
logging.error("failed to write_points.")
def read(self, query):
"""
Send a query to InfluxDB.
:param query (str) - the actual query string
"""
return self.client.query(query)
parser.add_argument("state", help="current task state")
parser.add_argument("weight", help="current task weight")
args = parser.parse_args()
task_name = args.task
task_state = args.state
task_weight = args.weight
influxdb_host = 'influxdb.influxdb.svc.cluster.local'
influxdb_port = 80
influxdb_user = '******'
influxdb_password = '******'
influxdb_database = 'student'
client = InfluxDBClient(influxdb_host, influxdb_port, influxdb_user,
influxdb_password, influxdb_database)
client.create_database(influxdb_database)
client.get_list_database()
client.switch_database(influxdb_database)
# delete old record
client.query('DROP MEASUREMENT "%s"' % (task_name))
current_time = datetime.now().isoformat(timespec='seconds')
json_body = [{
"measurement": "%s" % task_name,
"tags": {
"user": "******",
},
"time": "%s" % (str(current_time)),
"fields": {
class waterSensor():
sendLevel = True
maxLevel = 10
minLevel = 5
influxclient = None
currReading = 1
def connectInflux(self):
self.influxclient = InfluxDBClient(host='3.139.96.11',
port=8086,
database='waterSensor')
self.influxclient.create_database('waterSensor')
def sendReading(self):
if self.sendLevel:
json_body = [{
"measurement": "levels",
"tags": {
"host": "server01",
"region": "us-west"
},
"time": datetime.now(),
"fields": {
"waterLevel": self.currReading
}
}]
self.influxclient.write_points(json_body)
def on_connect(self, client, userdata, flags, rc):
print("Connected to server (i.e., broker) with result code " + str(rc))
#subscribe to topics of interest here
client.subscribe(
'lstsai/waterCtrl') #sub to led and set custom callback
client.message_callback_add('lstsai/waterCtrl',
self.waterCtrl_callback)
client.subscribe(
'lstsai/maxLevel') #sub to led and set custom callback
client.message_callback_add('lstsai/maxLevel', self.maxLevel_callback)
client.subscribe(
'lstsai/minLevel') #sub to led and set custom callback
client.message_callback_add('lstsai/minLevel', self.minLevel_callback)
def waterCtrl_callback(self, client, userdata, msg):
if str(msg.payload, "utf-8") == "start":
self.sendLevel = True
elif str(msg.payload, "utf-8") == "stop":
self.sendLevel = False
def maxLevel_callback(self, client, userdata, msg):
self.maxLevel = int(str(msg.payload, "utf-8"))
def minLevel_callback(self, client, userdata, msg):
self.minLevel = int(str(msg.payload, "utf-8"))
#Default message callback. Please use custom callbacks.
def on_message(self, client, userdata, msg):
print("on_message: " + msg.topic + " " + str(msg.payload, "utf-8"))
def main(self):
client = mqtt.Client()
client.on_message = self.on_message
client.on_connect = self.on_connect
client.connect(host="eclipse.usc.edu", port=11000, keepalive=60)
client.loop_start()
self.connectInflux()
#set the ports
ranger = 4
ledMax = 2
ledMin = 3
reading = 1
lastReading = 12 - grovepi.ultrasonicRead(ranger)
lastReading = 1
while True:
self.currReading = 12 - grovepi.ultrasonicRead(ranger)
if abs(self.currReading - lastReading) < 5:
self.sendReading()
client.publish("lstsai/waterLevel", self.currReading)
lastReading = self.currReading
if self.sendLevel:
self.sendReading()
if self.currReading > self.maxLevel:
grovepi.digitalWrite(ledMax, 1)
else:
grovepi.digitalWrite(ledMax, 0)
if self.currReading < self.minLevel:
grovepi.digitalWrite(ledMin, 1)
else:
grovepi.digitalWrite(ledMin, 0)
time.sleep(1)
class SmartModule(object):
"""Represents a HAPI Smart Module (Implementation).
Attributes:
id: ID of the site
name: Name of the site
wunder_key: Weather Underground key to be used
operator: Name of the primary site operator
email: Email address of the primary site operator
phone: Phone number of the primary site operator
location: Location or Address of the site
"""
def __init__(self):
self.mock = True
self.comm = communicator.Communicator(self)
self.data_sync = DataSync()
self.id = ""
self.name = ""
self.wunder_key = ""
self.operator = ""
self.email = ""
self.phone = ""
self.location = ""
self.longitude = ""
self.latitude = ""
self.scheduler = None
self.hostname = socket.gethostname()
self.last_status = ""
self.ifconn = None
self.rtc = rtc_interface.RTCInterface()
self.rtc.power_on_rtc()
self.launch_time = self.rtc.get_datetime()
self.asset = Asset(self.hostname)
self.asset.id = self.rtc.get_id()
self.asset.context = self.rtc.get_context()
self.asset.type = self.rtc.get_type()
self.ai = asset_interface.AssetInterface(self.asset.type,
self.rtc.mock)
self.rtc.power_off_rtc()
def load_influx_settings(self):
"""Load Influxdb server information stored in database base."""
try:
settings = {}
field_names = '''
server
port
username
password
'''.split()
sql = 'SELECT {fields} FROM influx_settings LIMIT 1;'.format(
fields=', '.join(field_names))
database = sqlite3.connect(utilities.DB_CORE)
db_elements = database.cursor().execute(sql).fetchone()
for field, value in zip(field_names, db_elements):
settings[field] = value
self.ifconn = InfluxDBClient(settings["server"], settings["port"],
settings["username"],
settings["password"])
Log.info("Influxdb information loaded.")
except Exception as excpt:
Log.exception("Trying to load Influx server information: %s.",
excpt)
finally:
database.close()
def become_broker(self):
"""If no broker found SM performs operation(s) to become the broker."""
try:
os.system("sudo systemctl start avahi-daemon.service"
) # We will change it soon!
except Exception as excpt:
Log.info("Error trying to become the Broker: %s.", excpt)
def find_service(self, zeroconf, service_type, name, state_change):
"""Check for published MQTT. If it finds port 1883 of type '_mqtt', update broker name."""
# Get the service we want (port 1883 and type '_mqtt._tcp.local.'
info = zeroconf.get_service_info(service_type, name)
if not (info.port == 1883 and service_type == "_mqtt._tcp.local."):
return
if state_change is ServiceStateChange.Added:
# If this is our service, update mqtt broker name and ip on self.comm (Communicator)
self.comm.broker_name = info.server
self.comm.broker_ip = str(socket.inet_ntoa(info.address))
elif state_change is ServiceStateChange.Removed:
# Implement way to handle removed MQTT service
# It only makes sense if leave zeroconf connection opened. It could be interesting.
pass
def find_broker(self, zeroconf):
"""Browser for our (MQTT) services using Zeroconf."""
browser = ServiceBrowser(zeroconf,
"_mqtt._tcp.local.",
handlers=[self.find_service])
def discover(self):
print(
"{status} Smart Module hosting asset {asset_id} {asset_type} {asset_context}."
.format(status="Mock" if self.rtc.mock else "Real",
asset_id=self.asset.id,
asset_type=self.asset.type,
asset_context=self.asset.context))
try:
max_sleep_time = 3 # Calling sleep should be reviewed.
zeroconf = Zeroconf()
Log.info("Performing Broker discovery...")
self.find_broker(zeroconf)
time.sleep(max_sleep_time
) # Wait for max_sleep_time to see if we found it.
if self.comm.broker_name or self.comm.broker_ip: # Found it.
Log.info("MQTT Broker: {broker_name} IP: {broker_ip}.".format(
broker_name=self.comm.broker_name,
broker_ip=self.comm.broker_ip))
else: # Make necessary actions to become the broker.
Log.info("Broker not found. Becoming the broker.")
self.become_broker()
time.sleep(max_sleep_time)
self.comm.connect() # Now it's time to connect to the broker.
except Exception as excpt:
Log.exception("[Exiting] Trying to find or become the broker.")
finally:
Log.info("Closing Zeroconf connection.")
zeroconf.close()
t_end = time.time() + 10
while (time.time() < t_end) and not self.comm.is_connected:
time.sleep(1)
self.comm.subscribe("SCHEDULER/RESPONSE")
self.comm.send("SCHEDULER/QUERY", "Where are you?")
Log.info("Waiting for Scheduler response...")
time.sleep(5) # Just wait for reply... Need a review?
self.comm.send("ANNOUNCE", self.hostname + " is online.")
t_end = time.time() + 2
while (time.time() < t_end) and not self.comm.is_connected:
time.sleep(1)
if not self.comm.scheduler_found: # Become the Scheduler (necessary actions as Scheduler)
try:
Log.info("No Scheduler found. Becoming the Scheduler.")
self.scheduler = Scheduler()
self.scheduler.smart_module = self
self.scheduler.prepare_jobs(self.scheduler.load_schedule())
self.comm.scheduler_found = True
self.comm.subscribe("SCHEDULER/QUERY")
self.comm.unsubscribe("SCHEDULER/RESPONSE")
self.comm.subscribe("STATUS/RESPONSE" + "/#")
self.comm.subscribe("ASSET/RESPONSE" + "/#")
self.comm.subscribe("ALERT" + "/#")
self.comm.send("SCHEDULER/RESPONSE", self.hostname)
self.comm.send("ANNOUNCE",
self.hostname + " is running the Scheduler.")
Log.info("Scheduler program loaded.")
except Exception as excpt:
Log.exception("Error initializing scheduler. %s.", excpt)
def load_site_data(self):
field_names = '''
id
name
wunder_key
operator
email
phone
location
longitude
latitude
'''.split()
try:
sql = 'SELECT {fields} FROM site LIMIT 1;'.format(
fields=', '.join(field_names))
database = sqlite3.connect(utilities.DB_CORE)
db_elements = database.cursor().execute(sql)
for row in db_elements:
for field_name, field_value in zip(field_names, row):
setattr(self, field_name, field_value)
Log.info("Site data loaded.")
except Exception as excpt:
Log.exception("Error loading site data: %s.", excpt)
finally:
database.close()
def connect_influx(self, database_name):
"""Connect to database named database_name on InfluxDB server.
Create database if it does not already exist.
Return the connection to the database."""
databases = self.ifconn.get_list_database()
for db in databases:
if database_name in db.values():
break
else:
self.ifconn.create_database(database_name)
self.ifconn.switch_database(database_name)
return self.ifconn
def push_sysinfo(self, asset_context, information):
"""Push System Status (stats) to InfluxDB server."""
timestamp = datetime.datetime.now()
conn = self.connect_influx(asset_context)
cpuinfo = [{
"measurement": "cpu",
"tags": {
"asset": self.name
},
"time": timestamp,
"fields": {
"unit": "percentage",
"load": information["cpu"]["percentage"]
}
}]
meminfo = [{
"measurement": "memory",
"tags": {
"asset": self.name
},
"time": timestamp,
"fields": {
"unit": "KBytes",
"free": information["memory"]["free"],
"used": information["memory"]["used"],
"cached": information["memory"]["cached"]
}
}]
netinfo = [{
"measurement": "network",
"tags": {
"asset": self.name
},
"time": timestamp,
"fields": {
"unit": "packets",
"packet_recv": information["network"]["packet_recv"],
"packet_sent": information["network"]["packet_sent"]
}
}]
botinfo = [{
"measurement": "boot",
"tags": {
"asset": self.name
},
"time": timestamp,
"fields": {
"unit": "timestamp",
"date": information["boot"]
}
}]
diskinf = [{
"measurement": "disk",
"tags": {
"asset": self.name
},
"time": timestamp,
"fields": {
"unit": "KBytes",
"total": information["disk"]["total"],
"free": information["disk"]["free"],
"used": information["disk"]["used"]
}
}]
tempinf = [{
"measurement": "internal",
"tags": {
"asset": self.name
},
"time": timestamp,
"fields": {
"unit": "C",
"unit temp": str(self.rtc.get_temp()),
}
}]
conn.write_points(cpuinfo + meminfo + netinfo + botinfo + diskinf +
tempinf)
def get_status(self):
"""Fetch system information (stats)."""
try:
sysinfo = SystemStatus(update=True)
return sysinfo
except Exception as excpt:
Log.exception("Error getting System Status: %s.", excpt)
def on_query_status(self):
"""It'll be called by the Scheduler to ask for System Status information."""
self.comm.send("STATUS/QUERY", "How are you?")
def on_check_alert(self):
"""It'll called by the Scheduler to ask for Alert Conditions."""
self.comm.send("ASSET/QUERY", "Is it warm here?")
def get_asset_data(self):
try:
self.asset.time = str(time.time())
self.asset.value = str(self.ai.read_value())
except Exception as excpt:
Log.exception("Error getting asset data: %s.", excpt)
self.asset.value = -1000
return self.asset.value
def log_sensor_data(self, data, virtual):
if not virtual:
try:
self.push_data(self.asset.name, self.asset.context,
self.asset.value, self.asset.unit)
except Exception as excpt:
Log.exception("Error logging sensor data: %s.", excpt)
else:
# For virtual assets, assume that the data is already parsed JSON
unit_symbol = {
'temp_c': 'C',
'relative_humidity': '%',
'pressure_mb': 'mb',
}
try:
for factor in ('temp_c', 'relative_humidity', 'pressure_mb'):
value = str(data[factor]).replace("%", "")
self.push_data(factor, "Environment", value,
unit_symbol[factor])
except Exception as excpt:
Log.exception("Error logging sensor data: %s.", excpt)
def push_data(self, asset_name, asset_context, value, unit):
try:
conn = self.connect_influx(asset_context)
json_body = [{
"measurement": asset_context,
"tags": {
"site": self.name,
"asset": asset_name
},
"time": str(datetime.datetime.now()),
"fields": {
"value": value,
"unit": unit
}
}]
conn.write_points(json_body)
Log.info("Wrote to analytic database.")
except Exception as excpt:
Log.exception("Error writing to analytic database: %s.", excpt)
def get_weather(self):
response = ""
url = ('http://api.wunderground.com/'
'api/{key}/conditions/q/{lat},{lon}.json').format(
key=self.wunder_key,
lat=self.latitude,
lon=self.longitude,
)
try:
f = urllib2.urlopen(url)
json_string = f.read()
parsed_json = json.loads(json_string)
response = parsed_json['current_observation']
f.close()
except Exception as excpt:
Log.exception("Error getting weather data: %s.", excpt)
return response
def log_command(self, job, result):
try:
now = str(datetime.datetime.now())
command = '''
INSERT INTO command_log (timestamp, command, result)
VALUES (?, ?, ?)
''', (now, job.name, result)
Log.info("Executed %s.", job.name)
database = sqlite3.connect(utilities.DB_HIST)
database.cursor().execute(*command)
database.commit()
except Exception as excpt:
Log.exception("Error logging command: %s.", excpt)
finally:
database.close()
def get_env(self):
now = datetime.datetime.now()
uptime = now - self.launch_time
days = uptime.days
minutes, seconds = divmod(uptime.seconds, utilities.SECONDS_PER_MINUTE)
hours, minutes = divmod(minutes, utilities.MINUTES_PER_HOUR)
s = ('''
Smart Module Status
Software Version v{version}
Running on: {platform}
Encoding: {encoding}
Python Information
- Executable: {executable}
- v{sys_version}
- location: {executable}
Timestamp: {timestamp}
Uptime: This Smart Module has been online for:
{days} days, {hours} hours, {minutes} minutes and {seconds} seconds.
''').format(
version=utilities.VERSION,
platform=sys.platform,
encoding=sys.getdefaultencoding(),
executable=sys.executable,
sys_version=sys.version.split()[0],
timestamp=now.strftime('%Y-%m-%d %H:%M:%S'),
days=days,
hours=hours,
minutes=minutes,
seconds=seconds,
)
s = utilities.trim(s) + '\n'
try:
self.comm.send("ENV/RESPONSE", s)
except Exception as excpt:
Log.exception("Error getting environment data: %s.", excpt)
from influxdb import InfluxDBClient
from influxdb import DataFrameClient
#server = Flask(__name__)
#server.config['INFLUXDB_DATABASE_URI'] = os.environ.get('DATABASE_URL', 'influxdb:///plan.db')
##server.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
#db = SQLAlchemy(server)
#db_URI = os.environ.get('DATABASE_URL', 'influxdb:///plan.db')
#engine = create_engine(db_URI)
client1 = InfluxDBClient(host='127.0.0.1',
port=8086) #(host='0.0.0.0', port=8086)
#client = InfluxDBClient(host='mydomain.com', port=8086, username='******', password='******', ssl=True, verify_ssl=True)
client1.create_database('plan')
#client.get_list_database()
client1.switch_database('plan')
#[{'fields': {'message': 'hii'}, 'measurement': 'table', 'tags': {'chat': <paho.mqtt.client.Client object at 0x7f196754ff10>}, 'time': '2020-05-20 14:27:27.884088'}]
def insert(messa):
print("insert=", messa)
client1.write_points(messa)
print(byenbiibbb)
#query3 = client1.query('select * from ' + 'hi' + ' where time < now()-' + '1d', chunked=True)
IPVcolumn = df['Ipv']
PPVcolumn = df['Ppv']
print("analyzed time values = \n",
Timeadjusted) # prints converted time column values
print(df)
# upload to influxdb cloud
host = 'localhost'
port = 8086
user = '******'
password = '******'
dbname = invertorID
# Temporarily avoid line protocol time conversion issues #412, #426, #431.
protocol = 'json'
client = InfluxDBClient(host, port, user, password, dbname)
client.create_database(dbname)
for i in range(newsize):
#print ('currentfile=',newsize)
json_body = [{
"measurement": invertorID,
"tags": {
"col3": "col3"
},
"time": Timeadjusted[i],
"fields": {
"temperature": tempcolumn[i],
"Eac_today": EACcolumn[i],
"Vpv": VPVcolumn[i],
"Iac": IACcolumn[i],
"Vac": VACcolumn[i],
def main():
global register
global measurement
global info
global status
global y_exp
global c_exp
global y_gen
global c_gen
global y_tot
global c_tot
global c_peak
global tmax
global min_ins
global c_stop
global c_start
global client
global flux_client
global ok
inverter_file = config.model
inverter = __import__(inverter_file)
if config.debug:
print("got it")
inverter_ip = inverter.inv_address()
print(inverter_ip)
if inverter_ip != "":
config.inverter_ip = inverter_ip
print(config.inverter_ip)
if config.debug:
print("statring setup")
print("opening modbus")
client = connect_bus(ip=config.inverter_ip,
PortN=config.inverter_port,
timeout=config.timeout)
if config.debug:
print("opening db")
try:
flux_client = InfluxDBClient(host=config.influxdb_ip,
port=config.influxdb_port,
username=config.influxdb_user,
password=config.influxdb_password)
except Exception as e:
print("main error: %s" % str(e))
flux_client = None
print("problem openning db")
if config.debug:
print("setting db")
flux_client.create_database(config.influxdb_database)
flux_client.create_database(config.influxdb_longterm)
if config.debug:
print("setting params")
last_loop = 0
info_time = 0
last_status = {}
y_exp = 0
c_exp = 0
y_tot = 0
c_tot = 0
y_gen = 0
c_gen = 0
c_peak = 0
c_start = ""
c_stop = ""
tmax = 0
min_ins = 1000
thread_time = [1]
thread_mean = 1
sleep = 0
midnight = (int(
time.mktime(
time.strptime(
time.strftime("%m/%d/%Y ") + " 00:00:00",
"%m/%d/%Y %H:%M:%S"))))
s = (
'SELECT %s FROM "%s" WHERE time >= %s and time <= %s and P_daily > 0'
) % (
'max("M_PExp") as "M_PExp", max("M_PTot") as "M_PTot", max("P_accum") as "P_accum", max("P_daily") as "P_daily", max("Temp") as "Temp"',
config.model, str(
(midnight - 24 * 3600) * 1000000000), str(midnight * 1000000000))
zeros = flux_client.query(s, database=config.influxdb_database)
m = list(zeros.get_points(measurement=config.model))
try:
y_exp = m[0]['M_PExp']
y_tot = m[0]['M_PTot']
y_gen = m[0]['P_daily']
tmax = m[0]['Temp']
except Exception as e:
print("main 1 error: %s" % str(e))
y_exp = 0
y_tot = 0
y_gen = 0
tmax = 0
s = (
'SELECT %s FROM "%s_info" WHERE time >= %s and time <= %s and Insulation > 0'
) % ('min("Insulation") as "Insulation"', config.model,
str((midnight - 3600 * 24) * 1000000000), str(midnight * 1000000000))
zeros = flux_client.query(s, database=config.influxdb_database)
m = list(zeros.get_points(measurement=config.model + "_info"))
try:
if config.debug:
print(m[0])
min_ins = m[0]['Insulation']
except Exception as e:
print("main 2 error: %s" % str(e))
if config.debug:
print(s)
min_ins = 1000
midnight = int(
time.mktime(
time.strptime(
time.strftime("%m/%d/%Y ") + " 23:59:59",
"%m/%d/%Y %H:%M:%S"))) + 1
if ((float(time.strftime("%S")) % 60) > 1):
time.sleep(59 - float(time.strftime("%S")))
utils.fill_blanks(flux_client, midnight)
forcast_time = forcast()
if config.debug:
print("loop")
ok = True
while ok:
current_time = time.time()
if (current_time - last_loop + thread_mean >= int(
config.scan_interval)):
last_loop = current_time
measurement = {}
info = {}
status = {}
j = 0
while do_map(client, config, inverter):
j += 1
if config.debug:
print(time.ctime(), register)
print("map looping")
time.sleep(10)
if j > 10:
if config.debug:
print("map infinite loop")
ok = False
if not ok:
return -1
if not ok:
return -1
current_time = time.time()
if (current_time - info_time > config.info_interval):
utils.write_influx(flux_client, info, config.model + "_info",
config.influxdb_database)
info_time = current_time
if config.debug:
print(info)
print(y_exp)
print(c_exp)
print(y_tot)
print(c_tot)
print(y_gen)
print(c_gen)
print(c_peak)
print(c_start)
print(c_stop)
print(tmax)
print(min_ins)
if (status != last_status):
utils.write_influx(flux_client, status, config.model + "_stat",
config.influxdb_database)
last_status = status
if config.debug:
print(status)
x = config.scan_interval - (float(time.strftime("%S")) %
config.scan_interval)
if x == 30:
x = 0
if (0.5 < x < 6):
time.sleep(x - 0.05)
utils.write_influx(flux_client, measurement, config.model,
config.influxdb_database)
if (config.supla_api != ""):
#read from supla
supla()
thread_time.insert(0, time.time() - last_loop)
if len(thread_time) > 5:
thread_time.pop()
thread_mean = sum(thread_time) / len(thread_time)
if (int(time.time()) > midnight):
daily = {}
daily['Insulation'] = float(min_ins)
min_ins = 1000
daily['Temp'] = float(tmax)
tmax = 0
s = 'SELECT cumulative_sum(integral("power90")) /3600 * 0.82 as power90, cumulative_sum(integral("power10")) /3600 * 0.82 as power10, cumulative_sum(integral("power")) /3600 * 0.82 as power FROM "forcast" WHERE time > now() -22h group by time(1d)'
zeros = flux_client.query(s, database=config.influxdb_database)
m = list(zeros.get_points(measurement="forcast"))
daily['f_power'] = float(m[0]['power'])
try:
daily['f_power90'] = float(m[0]['power90'])
daily['f_power10'] = float(m[0]['power10'])
except Exception as e:
print("main 3 error: %s" % str(e))
daily['f_power90'] = float(m[0]['power'])
daily['f_power10'] = float(m[0]['power'])
if config.debug:
print(midnight)
print(c_gen)
print(y_gen)
print(c_peak)
if y_exp != 0 and y_tot != 0:
daily['P_Load'] = float(c_gen - (c_exp - y_exp) +
(c_tot - y_tot))
daily['P_daily'] = float(c_gen)
c_gen = 0
if (y_exp != 0):
daily['P_Exp'] = float(c_exp - y_exp)
y_exp = c_exp
if (y_tot != 0):
daily['P_Grid'] = float(c_tot - y_tot +
config.extra_load * 24 / 1000)
y_tot = c_tot
daily['P_peak'] = float(c_peak)
daily['Start'] = c_start
daily['Shutdown'] = c_stop
utils.write_influx(flux_client, daily, config.model + "_daily",
config.influxdb_longterm,
(midnight - 24 * 3600) * 1000000000)
if config.debug:
print(time.ctime(midnight - 24 * 3600))
print(daily)
utils.send_measurements(midnight - 24 * 3600, midnight,
flux_client)
midnight = int(
time.mktime(
time.strptime(
time.strftime("%m/%d/%Y ") + " 23:59:59",
"%m/%d/%Y %H:%M:%S"))) + 1
if (int(time.time()) > forcast_time):
forcast_time = forcast()
else:
sleep = config.scan_interval - (
float(time.strftime("%S")) %
config.scan_interval) - thread_mean * 1.1
if sleep == config.scan_interval:
sleep = 0.0
# sleep = 0.95 * (config.scan_interval - (time.time() - last_loop)) + 0.05
if ((int(time.time() + sleep)) > midnight):
sleep = midnight - int(time.time())
if sleep < 0.16:
sleep = 0.1
# if sleep < 0:
# sleep = 1
time.sleep(sleep)
try:
pass
except Exception as e:
print("main 4 error: %s" % str(e))
ok = False
return -1
class influxIO(object):
try:
## Import fehlgeschlagen
if (PrivateImportError):
raise IOError(PrivateImportError)
if (ImportError):
raise IOError(ImportError)
# #################################################################################################
# # Funktion: ' Constructor '
## \details Die Initialisierung der Klasse KeepAlive
# \param[in] self der Objectpointer
# \param[in] interval
# \param[in] mqttClient
# \param[in] portal_id
# \return -
# #################################################################################################
def __init__(self, _host, _port, _username, _password, _database,
_gzip, logger):
#self.log = logger.getLogger('InfluxHandler')
self.host = _host
self.port = _port
self.username = _username
self.password = _password
self.database = _database
self.gzip = _gzip
self.influxdb_client = None
# # Ende Funktion: ' Constructor ' ################################################################
# #################################################################################################
# # Funktion: ' Destructor '
# #################################################################################################
#def __del__(self):
# # Ende Funktion: ' Destructor ' #################################################################
# #################################################################################################
# # Funktionen
# # Prototypen
# # if __name__ == '__main__':
# #################################################################################################
# #################################################################################################
# # Funktion: '_init_influxdb_database '
## \details Initialisiert die vorhandene Database, bzw. erzeugt eine neue.
# \param[in] -
# \return -
# #################################################################################################
def _init_influxdb_database(self, _database, callee):
# close connection if reload
if self.influxdb_client is not None:
self.influxdb_client.close()
time.sleep(1) #Login
ver = None
try:
self.influxdb_client = InfluxDBClient(host=self.host,
port=self.port,
username=self.username,
password=self.password,
database=_database,
gzip=self.gzip)
ver = self.influxdb_client.ping()
print("InfluxDB Version: {}".format(ver))
databases = self.influxdb_client.get_list_database()
if (len(
list(
filter(lambda x: x['name'] == _database,
databases))) == 0):
print("Erstelle Datenbank: {}".format(_database))
self.influxdb_client.create_database(_database)
if (callee == 'VrmGetData'):
print("Setzte Retention Policy: {}".format(_database))
#self.influxdb_client.alter_retention_policy('daily', database = self.database, duration = '52w', replication = 0, default = True)
#self.influxdb_client.create_retention_policy('sechs_monate', database = _database, duration = '26w', replication = 1, default = True)
self.influxdb_client.create_retention_policy(
'daily',
database=_database,
duration='52w',
replication=1,
default=True)
#print("Setzte Continuous query: {}".format(_database))
#select_clause = 'SELECT mean("AcPvOnGridPower") INTO "PvInvertersAcEnergyForwardDay" FROM "system" WHERE ("instance" = "Gateway") GROUP BY time(1d)'
#self.influxdb_client.create_continuous_query('PvDay', select_clause, _database, 'EVERY 10s FOR 1d')
self.influxdb_client.switch_database(_database)
print("{} initialisiert die Datenbank: {}".format(
callee, _database))
if (callee == 'VrmGetData'):
policyList = self.influxdb_client.get_list_retention_policies(
database=_database)
print("Retention Policies: {}".format(policyList))
queryList = self.influxdb_client.get_list_continuous_queries(
)
print("Continuous query: {}".format(queryList))
except requestException.ConnectionError as e:
print("ConnectionError (Init) : {}".format(e))
#for info in sys.exc_info():
# print("{}".format(info))
except:
print("Start Sequenz (Init)")
for info in sys.exc_info():
print("{}".format(info))
print("Ende Sequenz\nSonstiger Error")
return ver
# ping() tested die verbindung und gibt die influx version zurück.
# close() schließt den http Socket,
# influx startet die console
# DROP DATABASE EnergyAnzeige löscht diese
# # Ende Funktion: ' _init_influxdb_database ' ####################################################
# #################################################################################################
# # Funktion: '_isEmpty_influxdb_database '
## \details leere DatenBank
# \param[in] -
# \return -
# #################################################################################################
def _isEmpty_influxdb_database(self):
try:
query = "SELECT count(*) FROM {}./.*/".format(
_conf.INFLUXDB_DATABASE)
print(query)
result1 = self.influxdb_client.query(query)
print(result1)
except:
for info in sys.exc_info():
print("{}".format(info))
return False
# # Ende Funktion: ' _isEmpty_influxdb_database ' ####################################################
# #################################################################################################
# # Funktion: '_send_sensor_data_to_influxdb '
## \details
# \param[in] sensor_data
# \return -
# #################################################################################################
def _send_sensor_data_to_influxdb(self, sensor_data):
json.json_body = [{
"measurement": sensor_data.device, # pvinverter
"tags": {
"instance": sensor_data.instance # Piko, SMA
},
"fields": { # AcEnergyForwardDaySoFar: 1000.0
},
"time": sensor_data.timestamp
}]
jsDict = {}
strFields = ''
valueCnt = 0
for type, value in zip_longest(sensor_data.type,
sensor_data.value):
jsDict.update({type: value})
strFields = strFields + " " + str(type) + "=" + str(value)
valueCnt += 1
json.json_body[0]["fields"] = jsDict
retVal = False
try:
retVal = self.influxdb_client.write_points(json.json_body)
except requestException.ChunkedEncodingError as e:
print(
"ChunkedEncodingError (Write): {}\nAnzahl Daten: {} e: {}".
format(json.json_body, valueCnt, e))
#for info in sys.exc_info():
# print("{}".format(info))
except DbException.InfluxDBServerError as e:
print("ServerError (Write): {}\nAnzahl Daten: {} e: {}".format(
json.json_body, valueCnt, e))
#for info in sys.exc_info():
# print("{}".format(info))
except DbException.InfluxDBClientError as e:
print("CientError (Write): {}\nAnzahl Daten: {} e: {}".format(
json.json_body, valueCnt, e))
#for info in sys.exc_info():
# print("{}".format(info))
except requestException.ConnectionError as e:
print("ConnectionError (Write): {}\nAnzahl Daten: {} e: {}".
format(json.json_body, valueCnt, e))
#for info in sys.exc_info():
# print("{}".format(info))
except:
print("Start Sequenz")
for info in sys.exc_info():
print("{}".format(info))
print(
"Ende Sequenz\nSonstiger Error (Write): {}\nAnzahl Daten: {}"
.format(json.json_body, valueCnt))
return retVal
# # Ende Funktion: ' _send_sensor_data_to_influxdb ' ##############################################
# #################################################################################################
# # Funktion: '_Query_influxDb '
## \details Abfrage der Datenbank
# \param[in] -
# \return -
# #################################################################################################
def _Query_influxDb(self, queries, measurement, searchFor):
try:
retVal = []
points = []
results = []
errQuery = ''
errPoint = ''
errResult = ''
for query in queries:
errQuery = query
result = self.influxdb_client.query(query)
results.append(result)
for result in results:
errResult = result
point = list(result.get_points(measurement))
points.append(point)
for point in points:
errPoint = point
if (len(point) > 1):
for k in range(0, len(point)):
retVal.append(point[k][searchFor])
elif (len(point) > 0):
retVal.append(point[0][searchFor])
else:
retVal.append(0)
except:
for info in sys.exc_info():
print("{}".format(info))
print("errQuery: {}".format(errQuery))
print("errResult: {}".format(errResult))
print("errPoint: {}".format(errPoint))
return retVal
# # Ende Funktion: ' _Query_influxDb ' ####################################################
##### Fehlerbehandlung #####################################################
except IOError as e:
print(
'Eine der Bibliotheken konnte nicht geladen werden!\n{}!\n'.format(
e))
except:
for info in sys.exc_info():
print("Fehler: {}".format(info))
from influxdb import InfluxDBClient
import random
import time
client = InfluxDBClient(host='localhost', port=8086)
client.create_database('pract')
client.switch_database('pract')
VAL = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
fields = {x: 0 for x in VAL}
client.drop_measurement("events")
countStr = input("Введите количество итераций: ")
count = int(countStr)
i = 0
while (i < count):
i += 1
client.write_points([{"measurement": "events", "fields": fields}])
for a in fields:
fields[a] = fields[a] + random.randint(-2, 2)
time.sleep(1)
import asyncio
from datetime import datetime
from xknx import XKNX
from influxdb import InfluxDBClient
client = InfluxDBClient('localhost', 8086, 'root', 'root', 'metrics')
try:
client.create_database('metrics')
except Exception as e:
print(e)
def get_current_time():
return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
def send_metric_datapoint(measurement, location, value, timestamp):
json_body = [{
"measurement": measurement,
"tags": {
"location": location
},
"time": timestamp,
"fields": {
"value": value
}
}]
try:
client.write_points(json_body, database="metrics")
except Exception as e:
except subprocess.CalledProcessError:
print("Kapacitor is not yet ready. Checking again in 10 seconds.")
time.sleep(10)
continue
break
print("Kapacitor is ready to accept data.")
for line in args.lines:
for column in args.columns:
print("Running UDF for (" + str(line) + ", " + str(column) + ")")
client = InfluxDBClient(database = 'master')
client.drop_database('master')
initial_size = get_size()
client.create_database('master')
initial_time = datetime.now()
f = open(args.file, "r")
l = []
bucket_size = 100
if line >= 10000 or column >= 5000:
bucket_size = 5
read_line = []
for i in tqdm(range(line * 2)):
read_line.append( f.readline() )
for x in tqdm(range(line)):
s = read_line[x].split(" ")
v = {}
for y in range(column):
v["a" + str(y)] = float(s[y + 1])
class Output(cowrie.core.output.Output):
def __init__(self):
cowrie.core.output.Output.__init__(self)
def start(self):
try:
host = CONFIG.get('output_influx', 'host')
except Exception:
host = ''
try:
port = CONFIG.getint('output_influx', 'port')
except Exception:
port = 8086
try:
ssl = CONFIG.getboolean('output_influx', 'ssl')
except Exception:
ssl = False
self.client = None
try:
self.client = InfluxDBClient(host=host, port=port, ssl=ssl, verify_ssl=ssl)
except InfluxDBClientError as e:
log.err("output_influx: I/O error({0}): '{1}'".format(
e.errno, e.strerror))
return
if self.client is None:
log.err("output_influx: cannot instantiate client!")
return
if (CONFIG.has_option('output_influx', 'username') and
CONFIG.has_option('output_influx', 'password')):
username = CONFIG.get('output_influx', 'username')
password = CONFIG.get('output_influx', 'password', raw=True)
self.client.switch_user(username, password)
try:
dbname = CONFIG.get('output_influx', 'database_name')
except Exception:
dbname = 'cowrie'
retention_policy_duration_default = '12w'
retention_policy_name = dbname + "_retention_policy"
if CONFIG.has_option('output_influx', 'retention_policy_duration'):
retention_policy_duration = CONFIG.get(
'output_influx', 'retention_policy_duration')
match = re.search(r'^\d+[dhmw]{1}$', retention_policy_duration)
if not match:
log.err(("output_influx: invalid retention policy."
"Using default '{}'..").format(
retention_policy_duration))
retention_policy_duration = retention_policy_duration_default
else:
retention_policy_duration = retention_policy_duration_default
database_list = self.client.get_list_database()
dblist = [str(elem['name']) for elem in database_list]
if dbname not in dblist:
self.client.create_database(dbname)
self.client.create_retention_policy(
retention_policy_name, retention_policy_duration, 1,
database=dbname, default=True)
else:
retention_policies_list = self.client.get_list_retention_policies(
database=dbname)
rplist = [str(elem['name']) for elem in retention_policies_list]
if retention_policy_name not in rplist:
self.client.create_retention_policy(
retention_policy_name, retention_policy_duration, 1,
database=dbname, default=True)
else:
self.client.alter_retention_policy(
retention_policy_name, database=dbname,
duration=retention_policy_duration,
replication=1, default=True)
self.client.switch_database(dbname)
def stop(self):
pass
def write(self, entry):
if self.client is None:
log.err("output_influx: client object is not instantiated")
return
# event id
eventid = entry['eventid']
# measurement init
m = {
'measurement': eventid.replace('.', '_'),
'tags': {
'session': entry['session'],
'src_ip': entry['src_ip']
},
'fields': {
'sensor': self.sensor
},
}
# event parsing
if eventid in ['cowrie.command.failed',
'cowrie.command.input']:
m['fields'].update({
'input': entry['input'],
})
elif eventid == 'cowrie.session.connect':
m['fields'].update({
'protocol': entry['protocol'],
'src_port': entry['src_port'],
'dst_port': entry['dst_port'],
'dst_ip': entry['dst_ip'],
})
elif eventid in ['cowrie.login.success', 'cowrie.login.failed']:
m['fields'].update({
'username': entry['username'],
'password': entry['password'],
})
elif eventid == 'cowrie.session.file_download':
m['fields'].update({
'shasum': entry.get('shasum'),
'url': entry.get('url'),
'outfile': entry.get('outfile')
})
elif eventid == 'cowrie.session.file_download.failed':
m['fields'].update({
'url': entry.get('url')
})
elif eventid == 'cowrie.session.file_upload':
m['fields'].update({
'shasum': entry.get('shasum'),
'outfile': entry.get('outfile'),
})
elif eventid == 'cowrie.session.closed':
m['fields'].update({
'duration': entry['duration']
})
elif eventid == 'cowrie.client.version':
m['fields'].update({
'version': ','.join(entry['version']),
})
elif eventid == 'cowrie.client.kex':
m['fields'].update({
'maccs': ','.join(entry['macCS']),
'kexalgs': ','.join(entry['kexAlgs']),
'keyalgs': ','.join(entry['keyAlgs']),
'compcs': ','.join(entry['compCS']),
'enccs': ','.join(entry['encCS'])
})
elif eventid == 'cowrie.client.size':
m['fields'].update({
'height': entry['height'],
'width': entry['width'],
})
elif eventid == 'cowrie.client.var':
m['fields'].update({
'name': entry['name'],
'value': entry['value'],
})
elif eventid == 'cowrie.client.fingerprint':
m['fields'].update({
'fingerprint': entry['fingerprint']
})
# cowrie.direct-tcpip.data, cowrie.direct-tcpip.request
# cowrie.log.closed
# are not implemented
else:
# other events should be handled
log.err(
"output_influx: event '{}' not handled. Skipping..".format(
eventid))
return
result = self.client.write_points([m])
if not result:
log.err("output_influx: error when writing '{}' measurement"
"in the db.".format(eventid))
except Exception as e:
print('Could not parse value for key ' + key)
print(e)
if key_type == "string":
fields[key] = ''
elif key_type == "float":
fields[key] = 0.0
elif key_type == "int":
fields[key] = 0
return fields
client = InfluxDBClient(INFLUX_HOST, INFLUX_PORT, INFLUX_USER, INFLUX_PW,
INFLUX_DB)
client.create_database(INFLUX_DB)
c = Consumer({
'bootstrap.servers': KAFKA_BOOTSTRAP,
'group.id': KAFKA_GROUP_ID,
'default.topic.config': {
'auto.offset.reset': 'smallest'
},
'max.poll.interval.ms': 600000
})
#c.assign(TopicPartition('0', 0))
c.subscribe([KAFKA_TOPIC])
running = True
while running:
class InfluxDB:
"""
This class will support InfluxDB database connections and operations.
It includes methods to build the schema proposed to store on the database.
"""
db_client = None
table_results = None
def __init__(self, setting):
self.host = setting.database_info['host']
self.port = setting.database_info['port']
self.database = setting.database_info['database']
self.local_connection = setting.database_info['local_connection'].lower() \
in ['true', 'yes', 'y', '1']
if not self.local_connection:
self.username = setting.database_info['username']
self.password = setting.database_info['password']
self.ssl = setting.database_info['ssl'].lower() in [
'true', 'yes', 'y', '1'
]
self.verify_ssl = setting.database_info['verify_ssl'].lower() \
in ['true', 'yes', 'y', '1']
self.verbose_status = setting.verbose_status
self.setting = setting
def open_connection(self):
"""
Method in charge of the connection creation with the InfluxDB database.
In a local environment, it will remove the database content and
tables related to the application to keep test consistency.
:return:
"""
if self.local_connection:
# Local connection
self.db_client = InfluxDBClient(self.host, self.port,
self.database)
# Begin - This will happen only on local execution
if {'name': self.database} in self.db_client.get_list_database():
self.db_client.delete_series(self.database,
'telemetry_summary')
else:
self.db_client.create_database(self.database)
# End - This will happen only on local execution
else:
# Remote connection
self.db_client = InfluxDBClient(self.host,
self.port,
username=self.username,
password=self.password,
database=self.database,
ssl=self.ssl,
verify_ssl=self.verify_ssl)
self.db_client.create_database(self.database)
# It assures to be pointing to the target table after the connection
self.db_client.switch_database(self.database)
def fetch_results(self):
"""
Method in charge of retrieving all the information
from the database given the user's specification
:return:
"""
self.table_results = \
self.db_client.query('SELECT * '
'FROM "' + self.database + '"."autogen"."telemetry_summary"')
# It sorts/organizes the elements got from the database
if not isinstance(self.table_results, list):
# It creates a list based on the content retrieved from the database
self.table_results = list(
self.table_results.get_points(measurement='telemetry_summary'))
# InfluxDB uses its tags as keys on its built-in sorting process
# Switch ID tag is a string and intervenes on this process.
# In order to get the records in a proper order based on their
# insertion, we have to sort them based on the meta sequence number.
self.table_results = sorted(self.table_results,
key=lambda val:
(val['time'], val['service_id']))
else:
# It creates a list based on the content retrieved from the database
tmp = list()
for result in self.table_results:
tmp.append(
list(result.get_points(measurement='telemetry_summary')))
self.table_results = copy.deepcopy(tmp)
tmp.clear()
# It returns/visualizes the records resultant content
for table_result in self.table_results:
print(table_result)
def save_to_database(self, json_structure):
"""
Method in charge of the JSON structure
storing process to the database
:param json_structure:
:return: None
"""
# It saves the information to the database
written = self.db_client.write_points(json_structure)
if self.verbose_status:
if written:
print("Packet information saved to the database!")
else:
print("Errors saving the information to the database!")
def save_to(self, data):
"""
Method that supports the schema instantiation
to save teh JSON Dataset.
:param data: JSON data set with the telemetry information
:return: None
"""
try:
self.save_to_database(data)
if self.verbose_status:
# It will visualize all the information on the console
self.fetch_results()
except ValueError as value_error:
raise ValueError from value_error
def influxDB(org_id, contact, msg, sendtype="mail"):
client = InfluxDBClient(config.get("influxdb", "server"), config.get("influxdb", "port"), \
config.get("influxdb","user"), config.get("influxdb", "passwd"), config.get("influxdb","database"), timeout=2)
try:
client.create_database(config.get("influxdb", "database"))
except:
return False
try:
duration = msg["数据"][0]["故障时长"].split(">")[1].split("<")[0]
except:
return ("get duration failed")
try:
duration = convertDuration(duration)
except:
duration = 0.0
if "IP" in msg:
name = msg['IP'].split("\n")
else:
name = msg['名称'].split("\n")
ownerlist = sorted(contact.replace(mail_suffix, "").split(","))
owner = ",".join(ownerlist)
for item in name:
json_body = [{
"measurement": "alert",
"tags": {
"org": org_id,
"sendtype": sendtype,
"subject": msg['主题'],
"owner": owner,
"alerttype": msg['类型'],
"status": status,
"name": item,
"level": msg['严重性']
},
"fields": {
"value": 1,
"duration": duration,
}
}]
filter(json_body, client)
# 以个人为单位统计报警量
for item in ownerlist:
json_body = [{
"measurement": "ownercount",
"tags": {
"org": org_id,
"owner": item,
"sendtype": sendtype
},
"fields": {
"value": 1
}
}]
client.write_points(json_body)
# 报警量统计(以'\n'拆分name之后，导致发送量统计增多，因此这里单独把发送量拿出来统计)
json_body = [{
"measurement": "sendcount",
"tags": {
"org": org_id,
"sendtype": sendtype
},
"fields": {
"value": 1
}
}]
client.write_points(json_body)
class TweetsProducer(KafkaProducer):
'''
Tweets Producer class inheriting from the KafkaProducer class to
facilitate connection and interaction with a Kafka broker.
This class fetches a continuous stream of tweets from Twitter's API
and sends the text of these tweets into a Kafka topic for further processing.
Also it connects to InfluxDB as time-series database to store some
meta-data from these tweets.
'''
def __init__(self, topic, *args, **kwargs):
self.topic = topic
self.influxdb_host = kwargs.pop('influxdb_host', 'localhost')
self.influxdb_port = kwargs.pop('influxdb_port', 8086)
self.influxdb_database = kwargs.pop('influxdb_database', None)
self.influxdb_client = InfluxDBClient(host=self.influxdb_host,
port=self.influxdb_port,
username='******',
password='******',
database=self.influxdb_database)
self.influxdb_client.create_database(self.influxdb_database)
super().__init__(*args, **kwargs)
def produce(self, stream_url, params, auth):
"""
Stream a 1% sample from worldwide real-time tweets
See Twitter Labs sample-stream docs for more details
"""
response = requests.get(url=stream_url,
params=params,
auth=auth,
stream=True)
for line in response.iter_lines():
if line and line != b'Rate limit exceeded':
line = json.loads(line)
# Storing tweets' language
data_point = [{
# "timestamp":
"measurement": self.influxdb_database,
"tags": {
"language": line['data']['lang'],
},
"fields": {
"id": line['data']['id']
}
}]
if geo_data := line.get('includes', {}).get('places'):
# If tweet is tagged in a specific location
if coordinates := line['data'].get('geo', {}).get(
'coordinates', {}).get('coordinates'):
lon, lat = coordinates
# Else use the generic location of the tweet
else:
lon1, lat1, lon2, lat2 = geo_data[0]['geo']['bbox']
lon = (lon1 + lon2) / 2
lat = (lat1 + lat2) / 2
# Set country_code and place_name as tags so that we can
# filter and group by these values
data_point[0]["tags"]["country_code"] = geo_data[0][
"country_code"]
data_point[0]["tags"]["place_name"] = geo_data[0][
"full_name"]
# Latitudes and Longitudes should be set as fields to be
# retrieved with a SELECT statement
data_point[0]["fields"]["latitude"] = lat
data_point[0]["fields"]["longitude"] = lon
try:
self.influxdb_client.write_points(data_point)
logging.info("Successfully stored ID '{}'.".format(
line['data']['id']))
except (InfluxDBServerError, InfluxDBClientError) as e:
logging.info("Failed at storing ID '{}'. Error: {}".format(
line['data']['id'], e))
# Queueing tweets into Kafka for further processing
if line['data']['lang'] == 'en':
self.send(
self.topic,
json.dumps({
'id': line['data']['id'],
'tweet': line['data']['text']
}).encode())
logging.info("Queued tweet '{}'.".format(
line['data']['id']))
import openpyxl, serial
import numpy as np
from influxdb import InfluxDBClient
from time import time
from time import sleep
# Configuración Base de Datos
client = InfluxDBClient(host='localhost',
port=8086,
username='******',
password='******')
client.create_database('Lighting')
mn1 = 'Lamp1'
mn2 = 'Lamp2'
mn3 = 'Lamp3'
mn4 = 'Lamp4'
mn5 = 'Lamp5'
mn6 = 'Lamp6'
mn7 = 'Lamp7'
mn8 = 'Lamp8'
mn9 = 'Lamp9'
mn10 = 'Window1'
mn11 = 'Window2'
data = []
# Cargar constantes de luminarias
doc = openpyxl.load_workbook('Constantes2.xlsx')
hoja = doc.get_sheet_by_name('Hoja1')
kij = np.zeros((9, 9))
#main
if __name__ == '__main__':
parser = get_args_parser()
args = parser.parse_args()
if args.help:
parser.print_help()
parser.exit()
try:
db = InfluxDBClient(args.host,args.port,args.user,args.password)
db.get_database_list()
except Exception, err:
logging.exception(err)
print err
sys.exit()
try:
db.create_database(args.database)
except:
print "Error creating database"
db.switch_db(args.database)
#Generate dummy data
backMilliseconds = 86000 * 1
startTime = int(datetime.datetime.now().strftime('%s')) - backMilliseconds
timeInterval = 60 * 1
eventTypes = ["click", "view", "post", "comment"]
cpuSeries = {
'name': 'cpu_idle',
'columns': ['time','value','value1','value2','value3','host'],
'points': []
}
eventSeries = {
'name': "customer_events",
def send_influx_metrics(freezer):
try:
current_time = datetime.datetime.utcnow()
if freezer.COMP_STATE == 1:
COMP_STATE = "ON"
else:
COMP_STATE = "OFF"
logger.info("Sending metrics to InfluxDB")
client = InfluxDBClient(host=config.INFLUXDB["HOST"], port=8086, database='freezer', timeout=5)
json_body = [
{
"measurement": "temperature",
"tags": {
"type": "i2c",
"environment": config.ENVIRONMENT,
},
"time": current_time,
"fields": {
"value": freezer.TEMP1
}
},
{
"measurement": "temperature",
"tags": {
"type": "1w",
"environment": config.ENVIRONMENT,
},
"time": current_time,
"fields": {
"value": freezer.TEMP2
}
},
{
"measurement": "humidity",
"tags": {
"type": "i2c",
"environment": config.ENVIRONMENT,
},
"time": current_time,
"fields": {
"value": freezer.HUMIDITY
}
},
{
"measurement": "compressor",
"tags": {
"environment": config.ENVIRONMENT,
"state": COMP_STATE
},
"time": current_time,
"fields": {
"value": 1
}
}
]
client.create_database("freezer")
client.write_points(json_body)
except Exception as e:
logger.error("An ERROR occured while sending metrics to InfluxDB")
print(e)
pass
#!/usr/bin/env python3
from influxdb import InfluxDBClient
'''
InfluxDB database initialization
'''
client = InfluxDBClient('localhost', 8086, 'root', 'root', 'racktor')
client.create_database('racktor')
client.create_user('racktor', 'racktor')
client.grant_privilege('all', 'racktor', 'racktor')
client.close()
