def init_dbase():
"""
:param none
:return: none
"""
print("**********************************")
print("* INIT DBASE USERS *")
print("**********************************")
currentpathdir = os.path.dirname(os.path.realpath(__file__))
cred_file = os.path.join(currentpathdir, "credential.txt")
try:
data_json = get_json_data_from_file(cred_file)
except:
print("**********************************")
print("* Error : no file credential.txt *")
print("**********************************")
return
client = InfluxDBClient('localhost', 8086)
print("**********************************")
print("* CREATE INFLUXDB GRAFANA USER *")
print("**********************************")
# create a user for grafana
db_user = data_json['DATABASE_USER_READER']
db_password = data_json['DATABASE_PASSWORD_READER']
client.create_user(db_user, db_password, admin=False)
print("**********************************")
print("* CREATE INFLUXDB ADMIN USER *")
print("**********************************")
# create admin user with read/write privilege
db_user = data_json['DATABASE_USER_ADMIN']
db_password = data_json['DATABASE_PASSWORD_ADMIN']
client.create_user(db_user, db_password, admin=True)
def create_influxdb_user(self):
client = InfluxDBClient(**INFLUXDB['default'])
try:
client.create_user(self.name, self.password, admin=False)
except:
client.drop_user(self.name)
client.create_user(self.name, self.password, admin=False)
print("Create user: " + self.name)
def create_influx_account(username, password):
try:
client = InfluxDBClient(
host=settings.DATABASES['influx']['HOST'],
port=settings.DATABASES['influx']['PORT'],
username=settings.DATABASES['influx']['ADMIN_USER'],
password=settings.DATABASES['influx']['ADMIN_PASS']
)
# test connection
client.request(url='ping', expected_response_code=204)
client.create_user(username, password)
client.grant_privilege('all', settings.DATABASES['influx']['NAME'], username)
# TODO: add retention policy
client.close()
except (ConnectionError, InfluxDBClientError, InfluxDBServerError, Exception) as exc:
logger.error('Cannot connect to influxdb server -> {0}:{1}'.format(settings.DATABASES['influx']['HOST'],
settings.DATABASES['influx']['PORT']))
logger.error('Influx user and database could not be created for {0}'.format(username))
raise InfluxUserNotCreated
def setup_influx():
client = InfluxDBClient('localhost', 8086)
client.create_database(ANALYSIS_DB)
client.create_database(GLOBAL_DB)
for user in client.get_list_users():
if user['user'] == USER:
return
client.create_user(USER, PASS)
client.grant_privilege('all', ANALYSIS_DB, USER)
client.grant_privilege('all', GLOBAL_DB, USER)
client.create_retention_policy(SENTIMENT,
'30d',
1,
default=True,
database=ANALYSIS_DB)
client.create_retention_policy(GSENTIMENT,
'180d',
1,
default=True,
database=GLOBAL_DB)
INFLUX_DB_PASS = os.environ.get("NETWATCH_INFLUX_DB_PASS")
cmds = [
['pip', 'install', '-r', 'requirements.txt'],
['bash', './scripts/init.sh'],
['bash', './scripts/install_influx.sh'],
['bash', './scripts/install_grafana.sh'],
]
for cmd in cmds:
subprocess.run(cmd)
from influxdb import InfluxDBClient
i = 0
while i < MAX_RETRIES:
try:
client = InfluxDBClient(host='localhost',
port=8086,
username=INFLUX_DB_ADMIN_USER,
password=INFLUX_DB_ADMIN_PASS)
client.create_database("internet_speed")
break
except Exception as e:
print(f"Connection failed, retrying in {i ** 2 + 1} seconds")
time.sleep(i**2 + 1)
i += 1
client.create_user(INFLUX_DB_USER, INFLUX_DB_PASS, admin=False)
client.grant_privilage("all", "internet_speed", INFLUX_DB_USER)
class DataLogging:
_local = True
_hostname = ''
_port = 8086
_influx = None
_connection_ok = False
def __init__(self, hostname='', port=8086):
# Check for localhost
if hostname != '':
# Running remotely:
self._local = False
# Truncate protocol at start of hostname, if exists.
if hostname.startswith('http://'):
hostname = hostname[7:]
elif hostname.startswith('https://'):
hostname = hostname[8:]
self._hostname = hostname
self._port = port
# Attempt the connection to see if properties exist, and create them if not.
if not self._local:
self._init_influx_client()
def _init_influx_client(self):
try:
utils.validate_can_write_file(_DB_FAILED_WRITES)
# Connect to the server
print(f'Attempting connection to host \'{self._hostname}:{self._port}\'')
self._influx = InfluxDBClient(host=self._hostname, port=self._port, timeout=_DB_TIMEOUT)
# Check for the user existence
user_found = False
for user in self._influx.get_list_users():
if user['user'] == DB_USER:
user_found = True
if not user_found:
utils.v_print('Influx user not found, creating...')
self._influx.create_user(username=DB_USER, password=DB_PASS)
else:
utils.v_print('Identified Influx User...')
# Check for the database existence
db_found = False
for db in self._influx.get_list_database():
if db['name'] == DB_TABLE:
db_found = True
if not db_found:
utils.v_print('Influx table not found, creating...')
self._influx.create_database(dbname=DB_TABLE)
else:
utils.v_print('Identified Influx Collection...')
# Applying database permissions to the user
if (not user_found) or (not db_found):
self._influx.grant_privilege(privilege='ALL', database=DB_TABLE, username=DB_USER)
# Check for a retention policy
# TODO Implement some sort of retention policy.
pass
# Reset the connection with appropriate user data.
self._influx.close()
self._connect()
except ReqConnectionError:
self._connection_ok = False
print('Failed to connect, reverting to local backup until connection can be initialised.')
def _connect(self):
self._influx = InfluxDBClient(host=self._hostname,
port=self._port,
username=DB_USER,
password=DB_PASS,
database=DB_TABLE,
timeout=_DB_TIMEOUT)
# Update the status of the db connection.
if self._influx.ping():
self._connection_ok = True
print(f'Connection to {self._hostname}:{self._port} successful.')
self._check_repost_unsent_values()
else:
raise ReqConnectionError('Unable to ping the database.')
def log_sensor_output(self, data: utils.DataCapture):
if not self._local:
if self._connection_ok:
self._write_remote(data)
else:
self._init_influx_client()
if self._connection_ok:
self._check_repost_unsent_values()
self._write_remote(data)
else:
self._write_local(data)
print(data)
def _check_repost_unsent_values(self):
if not utils.validate_file_exists(_DB_FAILED_WRITES):
return
utils.v_print('Had values which were not successfully sent.')
all_sent_ok = True
for data in self._load_locals():
if self._connection_ok:
utils.v_print(f"ReSending dropped data: {data}")
self._write_remote(data)
else:
all_sent_ok = False
self._write_local(data)
if all_sent_ok:
print('Successfully pushed, all previously failed db writes, to server.')
os.remove(_DB_FAILED_WRITES)
def _write_remote(self, data: utils.DataCapture):
try:
self._influx.write_points([
{
"time": f'{datetime.fromtimestamp(data.timestamp).strftime("%Y-%m-%dT%T.%f")[:-3]}Z',
"measurement": "AQ",
"tags": {
"runID": _PROG_RUN_ID,
"hostname": _HOST_NAME
},
"fields": {
"temperature": data.temperature,
"humidity": data.humidity,
"pressure": data.pressure,
"gas": data.gas,
"quality": data.iaq_index
}
}
])
except Exception as err:
utils.v_print(err)
self._influx.close()
self._connection_ok = False
self._write_local(data)
def _write_local(self, data: utils.DataCapture):
self._write_locals([data])
@staticmethod
def _write_locals(data: [utils.DataCapture]):
with open(_DB_FAILED_WRITES, 'ab') as db_backups:
for val in data:
pickle.dump(val, db_backups)
@staticmethod
def _load_locals() -> [utils.DataCapture]:
backups = []
with open(_DB_FAILED_WRITES, 'rb') as db_backups:
while True:
try:
backups.append(pickle.load(db_backups))
except EOFError:
break
return backups
def shutdown(self):
if not self._local:
if self._connection_ok:
self._influx.close()
print("\n")
#Logic for user and DB check.
#Do you have a user? NO! Then create one!
if len(listUsers) == 0:
print("There is no user yet, create one"),
Question = "Create a new user with Administrator Privileges?"
reply = str(input(Question + ' (y/n): ')).lower().strip()
if reply[0] == "y":
#New user is Admin user!
Username = input()
Password = input()
Username_quoted = "'" + Username + "'"
Password_quoted = "'" + Password + "'"
dbClient.create_user(Username_quoted, Password_quoted, admin=True),
#Do you have a Db available (apart from _internal?) NO! Then create one!
if len(listDB) <= 1:
print("There is no DB yet, create one"),
db_name = input("Insert db name: "),
dbClient.create_database(db_name),
dbClient.switch_database(db_name)
if reply[0] == "n":
#New user is Basic user!
Username = input()
Password = input()
Username_quoted = "'" + Username + "'"
Password_quoted = "'" + Password + "'"
dbClient.create_user(Username_quoted, Password_quoted, admin=False),
#Do you have a Db available (apart from _internal?) NO! Then create one!
if len(listDB) <= 1:
#!/usr/bin/env python
from influxdb import InfluxDBClient
import os
my_env = os.environ.copy()
DB_NAME = 'enerlyzer'
client = InfluxDBClient('localhost', 8086, 'root', 'root', DB_NAME)
client.create_user('admin', my_env["MY_PASSWORD"], admin=True)
client.create_user('influx_user', my_env["INFLUX_USER_PASSWORD"], admin=False)
client.create_user('influx_reader',
my_env["INFLUX_READER_PASSWORD"],
admin=False)
client.create_database(DB_NAME)
client.grant_privilege('ALL', DB_NAME, 'influx_user')
client.grant_privilege('READ', DB_NAME, 'influx_reader')
def main(args=None):
print('\n--------------------------------------------- ')
print('-------- ble_sensors version: {:} -------- '.format(
pkg_resources.require("ble_sensors")[0].version))
print('--------------------------------------------- \n')
# Check that config file exists
# ble_sensor_config.create_config_file()
config = ble_sensor_config.get_config()
if config['InfluxDB'].getboolean('enabled') == True:
info('InfluxDB enabled!')
host = config['InfluxDB']['address']
try:
port = int(config['InfluxDB']['port'])
database = config['InfluxDB']['database']
username = config['InfluxDB']['username']
password = config['InfluxDB']['password']
influxDB_client = influxDB.connect_to_server(
host, port, username, password)
influxDB_client = influxDB.connect_to_database(
influxDB_client, database)
except ValueError:
error('Not valid InfluxDB port at influxDB config')
influxDB_client = None
else:
info('InfluxDB not configured!')
influxDB_client = None
parser = get_parser()
args = parser.parse_args(args)
if args.find_action:
find_ruuvitags(config)
elif not args.scan_time == None:
read_data_from_ruuvitags(timeout=args.scan_time,
influxDB_client=influxDB_client)
elif args.config_edit_action:
subprocess.call(['nano', config_file])
elif args.add_to_crontab:
info('Writing ble-sensor scanning job to crontab')
cron = CronTab(user='******')
iter = cron.find_comment('Scanning job for ble-sensors')
jobs = [i for i in iter]
if len(jobs) > 0:
warning('ble-sensor scanning job already found from crontab')
else:
job = cron.new(command='/usr/bin/python3 -m ble_sensors -r 10',
comment='Scanning job for ble-sensors')
job.minute.every(1)
cron.write()
info('Job written successfully to crontab!')
elif args.remove_from_crontab:
info('Trying to delete ble-sensor scanning job from crontab')
cron = CronTab(user='******')
iter = cron.find_comment('Scanning job for ble-sensors')
jobs = [i for i in iter]
if len(jobs) > 0:
for job in jobs:
temp = job.comment
cron.remove(job)
cron.write()
info('{} removed from crontab'.format(temp))
else:
warning('ble-sensor scanning job not found from crontab!')
elif args.install_action:
if query_yes_no('Do you want to install Bluez protocol stack?'):
info('Installing Bluez protocol stack...\n')
subprocess.call(
['sudo', 'apt-get', 'install', 'bluez', 'bluez-hcidump'])
print()
info('Bluez protocol stack installed!')
print()
if query_yes_no('Do you want to search for new sensors?'):
print()
find_ruuvitags(config)
if query_yes_no(
'Do you want to install inluxDB server to this device?'):
info('Installing inluxDB server...\n')
command = 'wget -qO- https://repos.influxdata.com/influxdb.key | sudo apt-key add - source /etc/os-release'.split(
' ')
subprocess.call(command)
command = 'test $VERSION_ID = "7"'.split(' ')
subprocess.call(command)
command = 'echo "deb https://repos.influxdata.com/debian wheezy stable" | sudo tee /etc/apt/sources.list.d/influxdb.list'.split(
' ')
subprocess.call(command)
command = 'test $VERSION_ID = "8"'.split(' ')
subprocess.call(command)
command = 'echo "deb https://repos.influxdata.com/debian jessie stable" | sudo tee /etc/apt/sources.list.d/influxdb.list'.split(
' ')
subprocess.call(command)
command = 'test $VERSION_ID = "9"'.split(' ')
subprocess.call(command)
command = 'echo "deb https://repos.influxdata.com/debian stretch stable" | sudo tee /etc/apt/sources.list.d/influxdb.list'.split(
' ')
subprocess.call(command)
command = 'sudo apt-get update'.split(' ')
subprocess.call(command)
command = 'sudo apt-get install influxdb'.split(' ')
subprocess.call(command)
command = 'sudo systemctl unmask influxdb.service'.split(' ')
subprocess.call(command)
command = 'sudo systemctl start influxdb'.split(' ')
subprocess.call(command)
command = 'sudo systemctl enable influxdb.service'.split(' ')
subprocess.call(command)
command = 'sudo apt install influxdb-client'.split(' ')
subprocess.call(command)
# Generate password
dbname = 'measurements'
user = '******'
alphabet = string.ascii_letters + string.digits
password = ''.join(secrets.choice(alphabet) for i in range(20))
client = InfluxDBClient(host='localhost', port=8086)
client.create_database(dbname)
client.switch_database(database)
client.ping()
client.create_user(user, password, admin=True)
config.set('InfluxDB', 'enabled', 'True')
config.set('InfluxDB', 'address', 'localhost')
config.set('InfluxDB', 'port', str(8086))
config.set('InfluxDB', 'database', dbname)
config.set('InfluxDB', 'username', user)
config.set('InfluxDB', 'password', password)
with open(config_file, 'w') as configfile:
config.write(configfile)
print()
info('inluxDB installed!!')
print()
print(client)
if query_yes_no('Do you want to edit / set InfluxDB parameters?'):
print()
temp = input('Enable InfluxDB reporting: ')
config.set('InfluxDB', 'enabled', temp)
temp = input('Address: ')
config.set('InfluxDB', 'address', temp)
temp = input('Port: ')
config.set('InfluxDB', 'port', temp)
temp = input('Database: ')
config.set('InfluxDB', 'database', temp)
temp = input('Username: '******'InfluxDB', 'username', temp)
temp = input('Password: '******'InfluxDB', 'password', temp)
with open(config_file, 'w') as configfile:
config.write(configfile)
else:
parser.print_usage()
print('\n---------------------------------------------\n')
from influxdb import InfluxDBClient
client = InfluxDBClient(host='192.168.1.68', port=8086)
client.create_database('smartcity')
client.create_user("grafana", "")
class Store(object):
def __init__(self, host='localhost', port=8086):
self.host = host
self.port = port
self.connect()
def connect(self):
self.client = InfluxDBClient(host=self.host, port=self.port)
def setUser(self, user, pw):
self.client.create_user(user, pw)
self.client.switch_user(user, pw)
def setDatabase(self, dbname):
dbs = self.client.get_list_database()
print(dbs)
self.client.create_database(dbname) # if it already exists, doesn't give any error...
self.client.switch_database(dbname)
self.client.create_retention_policy('forever', 'INF', 3, dbname)
def put(self, measurement, fields, tags = None, time = None):
"""
measurement -> id of measurement
time -> default now, otherwise RFC RFC3339 value
"tags": {
"tag1": "server01",
"tag2": "us-west"
},
"time": "2009-11-10T23:00:00Z",
"fields": {
"field1": 0.64,
"field2": 3
}
"""
if time == None:
time = self.client.now()
json_body = [
{
"measurement": measurement,
"tags": tags,
"time": time,
"fields": fields
}
]
self.client.write_points(json_body)
def get(self, measurement, fields = None, tags = None, time_start=None, time_end=None):
"""
tags -> [tag1, tag2]
fields -> [field1, field2]
time_* -> RFC 3339 timestamp, i.e. '2017-04-13T14:34:23.111142+00:00'
"""
stags = ''
if (tags != None):
for tag in tags:
stags += '"' + tag + '"::tag,'
sfields = ''
if (fields != None):
for field in fields:
sfields += '"' + field + '"::field,'
sfields = sfields[:-1] # remove last comma
if (sfields == ''):
if (stags != ''):
stags = stags[:-1]
sfrom = 'FROM ' + measurement
shwere = ''
if (time_start != None) or (time_end != None):
swhere = 'WHERE '
sstart = ''
if (time_start != None):
sstart = 'TIME > ' + time_start
send =''
if (time_end != None):
if (time_start != None):
sstart += ' AND '
send = 'TIME < ' + time_end
query = "SELECT " + stags + sfields + sfrom + shwere + sstart + send
result = self.client.query(query)
return (result.get_points())
def query(self, query):
result = self.client.query(query)
return (result.get_points())
#!/usr/bin/env python3
from influxdb import InfluxDBClient
'''
InfluxDB database initialization
'''
client = InfluxDBClient('localhost', 8086, 'root', 'root', 'racktor')
client.create_database('racktor')
client.create_user('racktor', 'racktor')
client.grant_privilege('all', 'racktor', 'racktor')
client.close()
class InfluxClient:
READ_PRIVILEGE = 'read'
TELEMETRY_CQ_FMT = """
CREATE CONTINUOUS QUERY "telemetry_{0}_cq" ON "{4}"
RESAMPLE EVERY {3}
BEGIN
SELECT
mean("batteryLevel") AS "batteryLevel",
mean("lightLevel") AS "lightLevel",
mean("rssi") AS "rssi",
mean("sensitivity") AS "sensitivity",
sum("singleClick") AS "singleClick",
sum("threshold") AS "threshold",
sum("doubleTap") AS "doubleTap",
mean("temperature") AS "temperature",
mean("humidity") as "humidity",
mean("x") AS "x",
mean("y") AS "y",
mean("z") AS "z",
max("history") AS "history",
MODE("sourceId") AS "sourceId"
INTO
"{1}"."telemetry_{0}"
FROM
"{2}"."telemetry"
GROUP BY time({0}), trackingId
END
"""
LOCATION_CQ_FMT = """
CREATE CONTINUOUS QUERY "locations_{0}_cq" ON "{4}"
RESAMPLE EVERY {3} FOR {5}
BEGIN
SELECT
mean("rssi") AS "rssi",
COUNT("rssi") AS "scans",
COUNT("rssi")/(-mean("rssi")) as "quality",
MODE("fSourceId") AS "fSourceId",
MODE("fTrackingId") AS "fTrackingId"
INTO
"{1}"."locations_{0}"
FROM
"{2}"."locations"
GROUP BY time({0}), trackingId, sourceId
END
"""
POSTIION_CQ_FMT = """
CREATE CONTINUOUS QUERY "positions_{0}_cq" ON "{4}"
RESAMPLE EVERY {3}
BEGIN
SELECT
mean("coord_latitude") AS "coord_latitude",
mean("coord_longitude") AS "coord_longitude"
INTO
"{1}"."positions_{0}"
FROM
current_rp.position
GROUP BY time({0}), trackingId
END
"""
REMOVE_CQ_FMT = """
DROP CONTINUOUS QUERY "{0}_{1}_cq" ON "{2}"
"""
def __init__(self, address, port, user_name, password):
self._client = InfluxDBClient(host=address.replace('http://', ''),
port=port,
username=user_name,
password=password)
def create_database(self, database_name):
print "Creating database %s" % database_name
self._client.create_database(database_name)
def create_user(self, user_name, password, database_name=None):
print "Creating user %s" % user_name
try:
self._client.create_user(user_name, password)
except InfluxDBClientError as e:
if e.message != 'user already exists':
raise e
if database_name is not None:
self._client.grant_privilege(self.READ_PRIVILEGE, database_name,
user_name)
def create_retention_policy(self, database_name, policy_name, duration):
print "Creating retention policy %s with duration %s on database %s" % (
policy_name, duration, database_name)
try:
self._client.create_retention_policy(policy_name,
duration,
1,
database=database_name)
except InfluxDBClientError as e:
if e.message == 'retention policy already exists':
print "Updating retention policy %s with duration %s on database %s" % (
policy_name, duration, database_name)
self._client.alter_retention_policy(policy_name, database_name,
duration, 1)
def recreate_continuous_query(self, database_name, aggregation_time,
retention_policy, source_retention_policy,
resample_time, resample_for):
self._execute_query(
self.REMOVE_CQ_FMT.format('telemetry', aggregation_time,
database_name), database_name)
self._execute_query(
self.REMOVE_CQ_FMT.format('locations', aggregation_time,
database_name), database_name)
self._execute_query(
self.REMOVE_CQ_FMT.format('positions', aggregation_time,
database_name), database_name)
self._execute_query(
self.TELEMETRY_CQ_FMT.format(aggregation_time, retention_policy,
source_retention_policy,
resample_time, database_name),
database_name)
self._execute_query(
self.LOCATION_CQ_FMT.format(aggregation_time, retention_policy,
source_retention_policy, resample_time,
database_name, resample_for),
database_name)
self._execute_query(
self.POSTIION_CQ_FMT.format(aggregation_time, retention_policy,
source_retention_policy, resample_time,
database_name), database_name)
def _execute_query(self, query, database_name):
print "Executing query %s" % query
self._client.query(query, database=database_name)
