def _connect(self):
"""
Connect to the influxdb server
"""
try:
# Open Connection
if influxdb.__version__ > "0.1.6":
self.influx = InfluxDBClient(self.hostname, self.port,
self.username, self.password,
self.database, self.ssl)
else:
self.influx = InfluxDBClient(self.hostname, self.port,
self.username, self.password,
self.database)
# Log
self.log.debug("InfluxdbHandler: Established connection to "
"%s:%d/%s.",
self.hostname, self.port, self.database)
except Exception, ex:
# Log Error
self._throttle_error("InfluxdbHandler: Failed to connect to "
"%s:%d/%s. %s",
self.hostname, self.port, self.database, ex)
# Close Socket
self._close()
return
class InfluxDBOutput(Thread):
output_name = 'influxdb'
def __init__(self, host, port, username, password, database, prefetch=10, proto='http'):
Thread.__init__(self)
self.prefetch = prefetch
self.proto = proto
self.host = host
self.port = port
if self.proto == 'http':
self.client = InfluxDBClient(host, port, username, password, database)
elif self.proto == 'udp':
self.client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
else:
raise ValueError('unknown protocol: %s' % (self.proto, ))
def set_in_queue(self, queue):
self._queue = queue
def run(self):
metrics = []
while True:
metrics += self._queue.get()
if len(metrics) >= self.prefetch:
try:
if self.proto == 'http':
self.client.write_points(metrics)
elif self.proto == 'udp':
self.client.sendto(json.dumps(metrics), (self.host, self.port))
except Exception as err:
logger.warning('error while processing event: %s' % (str(err), ))
finally:
metrics = []
def __init__(self, context, options):
super().__init__(context, options)
self._influxdb = InfluxDBClient(
host=self.options.host,
port=self.options.port,
username=self.options.username,
password=self.options.password,
database=self.options.database,
)
def getDataFromDb(slurm_start, slurm_end, user, database):
""" Connecting to Influx database """
start_date = encodeTime(slurm_start)
end_date = encodeTime(slurm_end)
client = InfluxDBClient(config.db_host, config.db_port, config.db_user,
config.db_pass)
qry_str = 'SELECT "value", "jobid" FROM ' + database + ' WHERE time > \'' + start_date + '\' AND time < \'' + end_date + '\' AND "user"=\'' + user + '\''
qry_res = client.query(qry_str)
return qry_res
def __init__(self, settings=None, database=None):
settings = deepcopy(settings) or {}
settings.setdefault('host', u'localhost')
settings.setdefault('port', u'8086')
settings.setdefault('username', u'root')
settings.setdefault('password', u'root')
settings.setdefault('database', database)
settings.setdefault('pool_size', 10)
settings.setdefault('use_udp', False)
settings.setdefault('udp_port', u'4444')
settings['port'] = int(settings['port'])
settings['udp_port'] = int(settings['udp_port'])
self.__dict__.update(**settings)
# Bookeeping for all databases having been touched already
self.databases_written_once = Set()
# Knowledge about all databases to be accessed using UDP
# TODO: Refactor to configuration setting
self.udp_databases = [
{
'name': 'luftdaten_info',
'port': u'4445'
},
]
self.host_uri = u'influxdb://{host}:{port}'.format(**self.__dict__)
log.info(u'Storage target is {uri}, pool size is {pool_size}',
uri=self.host_uri,
pool_size=self.pool_size)
self.influx_client = InfluxDBClient(host=self.host,
port=self.port,
username=self.username,
password=self.password,
database=self.database,
pool_size=self.pool_size,
timeout=10)
# TODO: Hold references to multiple UDP databases using mapping "self.udp_databases".
self.influx_client_udp = None
if settings['use_udp']:
self.influx_client_udp = InfluxDBClient(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
use_udp=settings['use_udp'],
udp_port=settings['udp_port'],
timeout=10)
class InfluxOutput(BaseOutput):
"""
Writes point into influxdb.
Each system metrics will be written into separated measurement.
All process metrics will be tagged by process_name and will be written into
separated metrics.
All points will be tagged with hostname.
"""
section_name = "influx"
hostname = socket.gethostname()
processname_pattern = re.compile(r"^process:([^\:]+):(.*)")
def init(self, **params):
self.influx_client = InfluxDBClient(**params)
supermann.utils.getLogger(self).info(
"Using InfluxOutput with params %s", str(params))
def __init__(self):
super(InfluxOutput, self).__init__()
self.bulk = []
def flush(self):
self.influx_client.write_points([p.to_influx() for p in self.bulk])
self.bulk = []
def event(self, **data):
try:
service = data["service"]
metric = data["metric_f"]
except:
return
if service.startswith("system"):
self.bulk.append(
Point(measurement=service,
point={"metric": metric},
tags={"hostname": self.hostname}))
elif service.startswith("process"):
processname, tail = self.processname_pattern.findall(service)[0]
self.bulk.append(
Point(measurement="process:{}".format(tail),
point={"metric": metric},
tags={
"hostname": self.hostname,
"process": processname
}))
class InfluxConnector:
client: InfluxDBClient
def __init__(self):
load_dotenv()
user=os.getenv("INFLUX_DB_USER")
password=os.getenv("INFLUX_DB_PASSWORD")
self.client = InfluxDBClient('localhost', 8086, user, password, 'pi_influxdb')
def get_client(self):
return self.client
def write_point(self,influx_point):
self.client.write_points([influx_point])
def create_client() -> InfluxDBClient:
"""
Create a client for the Influx database.
Includes connection test and database and retention policy creation.
:return: an instance of InfluxDBClient.
"""
client = InfluxDBClient(host=INFLUXDB_HOST,
port=INFLUXDB_PORT,
username=INFLUXDB_USERNAME,
password=INFLUXDB_PASSWORD,
database=INFLUXDB_DATABASE)
# Test connection
try:
client.ping()
except requests.exceptions.ConnectionError:
_logger.error('Connection to InfluxDB was refused!')
exit(code=1) # TODO replace with backoff retry
# Create database (will pass if already exists)
client.create_database(INFLUXDB_DATABASE)
# Set retention policy on database
client.create_retention_policy(name=RETENTION_NAME,
duration=RETENTION_DURATION,
replication=RETENTION_REPLICATION,
database=INFLUXDB_DATABASE,
default=True)
_logger.info("InfluxDB client created successfully")
return client
def configure(self, options, noseconfig):
""" Call the super and then validate and call the relevant parser for
the configuration file passed in """
super(Grafana, self).configure(options, noseconfig)
self.context = ContextHelper()
cfgifc = self.context.get_config()
self.duts = options.get(
'duts',
cfgifc.config.get('plugins', {}).get('_default_', {}))
self.first_time = True
o = options.influxdb
if o:
self.client = InfluxDBClient(o.host, o.port, o.user, o.password,
o.db)
def _setup_gzip_client(inst):
inst.cli = InfluxDBClient('localhost',
inst.influxd_inst.http_port,
'root',
'',
database='db',
gzip=True)
def __init__(self, settings=None, database=None):
settings = deepcopy(settings) or {}
settings.setdefault('host', u'localhost')
settings.setdefault('port', u'8086')
settings.setdefault('version', u'0.9')
settings.setdefault('username', u'root')
settings.setdefault('password', u'root')
settings.setdefault('database', database)
settings.setdefault('use_udp', False)
settings.setdefault('udp_port', u'4444')
settings['port'] = int(settings['port'])
settings['udp_port'] = int(settings['udp_port'])
self.__dict__.update(**settings)
self.databases_written_once = Set()
self.udp_databases = [
{
'name': 'luftdaten_testdrive',
'port': u'4445'
},
]
self.host_uri = u'influxdb://{host}:{port}'.format(**self.__dict__)
log.info(u'Storage target is {uri}', uri=self.host_uri)
self.influx_client = InfluxDBClient(host=self.host,
port=self.port,
username=self.username,
password=self.password,
database=self.database,
timeout=10)
# TODO: Hold references to multiple UDP databases using mapping "self.udp_databases".
self.influx_client_udp = None
if settings['use_udp']:
self.influx_client_udp = InfluxDBClient(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
use_udp=settings['use_udp'],
udp_port=settings['udp_port'],
timeout=10)
async def influx_fetch_limit_1(influx_clt: InfluxDBClient,
measurement,
is_near_now: bool,
database=None):
sql = f"SELECT * FROM {measurement} ORDER BY time {'DESC' if is_near_now else 'ASC'} LIMIT 1"
rs: ResultSet = influx_clt.query(sql)
points = list(rs.get_points(measurement))
if len(points) == 1:
return points[-1]
def __init__(self, settings=None, database=None):
settings = deepcopy(settings) or {}
settings.setdefault('host', u'localhost')
settings.setdefault('port', u'8086')
settings.setdefault('username', u'root')
settings.setdefault('password', u'root')
settings.setdefault('database', database)
settings.setdefault('use_udp', False)
settings.setdefault('udp_port', u'4444')
settings['port'] = int(settings['port'])
settings['udp_port'] = int(settings['udp_port'])
self.__dict__.update(**settings)
# Bookeeping for all databases having been touched already
self.databases_written_once = Set()
# Knowledge about all databases to be accessed using UDP
# TODO: Refactor to configuration setting
self.udp_databases = [
{'name': 'luftdaten_info', 'port': u'4445'},
]
self.host_uri = u'influxdb://{host}:{port}'.format(**self.__dict__)
log.info(u'Storage target is {uri}', uri=self.host_uri)
self.influx_client = InfluxDBClient(
host=self.host, port=self.port,
username=self.username, password=self.password,
database=self.database,
timeout=10)
# TODO: Hold references to multiple UDP databases using mapping "self.udp_databases".
self.influx_client_udp = None
if settings['use_udp']:
self.influx_client_udp = InfluxDBClient(
host=self.host, port=self.port,
username=self.username, password=self.password,
use_udp=settings['use_udp'], udp_port=settings['udp_port'],
timeout=10)
def get_client(name: str = "default") -> InfluxDBClient:
if name not in _clients:
if not getattr(settings, "INFLUX_DATABASES", None):
raise ImproperlyConfigured("INFLUX_DATABASES is not configured.")
if name not in settings.INFLUX_DATABASES:
raise ValueError("Unknown database: {}".format(name))
_clients[name] = InfluxDBClient(**settings.INFLUX_DATABASES[name])
return _clients[name]
def __init__(self, host, port, username, password, database, prefetch=10, proto='http'):
Thread.__init__(self)
self.prefetch = prefetch
self.proto = proto
self.host = host
self.port = port
if self.proto == 'http':
self.client = InfluxDBClient(host, port, username, password, database)
elif self.proto == 'udp':
self.client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
else:
raise ValueError('unknown protocol: %s' % (self.proto, ))
def __init__(self, settings=None, database=None):
settings = deepcopy(settings) or {}
settings.setdefault('host', u'localhost')
settings.setdefault('port', u'8086')
settings.setdefault('version', u'0.9')
settings.setdefault('username', u'root')
settings.setdefault('password', u'root')
settings.setdefault('database', database)
settings.setdefault('use_udp', False)
settings.setdefault('udp_port', u'4444')
settings['port'] = int(settings['port'])
settings['udp_port'] = int(settings['udp_port'])
self.__dict__.update(**settings)
self.databases_written_once = Set()
self.udp_databases = [
{'name': 'luftdaten_testdrive', 'port': u'4445'},
]
self.host_uri = u'influxdb://{host}:{port}'.format(**self.__dict__)
log.info(u'Storage target is {uri}', uri=self.host_uri)
self.influx_client = InfluxDBClient(
host=self.host, port=self.port,
username=self.username, password=self.password,
database=self.database,
timeout=10)
# TODO: Hold references to multiple UDP databases using mapping "self.udp_databases".
self.influx_client_udp = None
if settings['use_udp']:
self.influx_client_udp = InfluxDBClient(
host=self.host, port=self.port,
username=self.username, password=self.password,
use_udp=settings['use_udp'], udp_port=settings['udp_port'],
timeout=10)
def do_influx_request(n_points, host,
database): # type: (int, str, str) -> float
conn = InfluxDBClient(host=host)
conn.switch_database('p2')
points = generate_fake_data(n_points)
request_start_time = time.time()
conn.write_points(points, time_precision='u', database=database)
return time.time() - request_start_time
def _setup_influxdb_server(inst):
inst.influxd_inst = InfluxDbInstance(
inst.influxdb_template_conf,
udp_enabled=getattr(inst, 'influxdb_udp_enabled', False),
)
inst.cli = InfluxDBClient('localhost',
inst.influxd_inst.http_port,
'root',
'',
database='db')
if not using_pypy:
inst.cliDF = DataFrameClient('localhost',
inst.influxd_inst.http_port,
'root',
'',
database='db')
class Grafana(ExtendedPlugin):
"""
Log events to Grafana.
"""
enabled = False
score = 480 # Before email
def configure(self, options, noseconfig):
""" Call the super and then validate and call the relevant parser for
the configuration file passed in """
super(Grafana, self).configure(options, noseconfig)
self.context = ContextHelper()
cfgifc = self.context.get_config()
self.duts = options.get(
'duts',
cfgifc.config.get('plugins', {}).get('_default_', {}))
self.first_time = True
o = options.influxdb
if o:
self.client = InfluxDBClient(o.host, o.port, o.user, o.password,
o.db)
def setup_snmp_bigiq(self, devices):
for device in devices:
if device.kind != KIND_TMOS_BIGIQ:
continue
rstifc = self.context.get_icontrol_rest(device=device)
payload = SnmpInbound(contactInformation='', machineLocation='')
address = {'address': '0.0.0.0', 'mask': '0.0.0.0'}
payload.clientAllowList.append(address)
rstifc.api.put(SnmpInbound.URI,
payload=payload) # @UndefinedVariable
payload = SnmpV1V2cAccessRecords()
payload.update(community='public', addressType='IPv4')
rstifc.api.post(SnmpV1V2cAccessRecords.URI, payload=payload)
def startTest(self, test, blocking_context=None):
if self.first_time:
self.setup_snmp_bigiq(expand_devices(self.duts))
self.first_time = False
def graphite_start(self):
LOG.info('Graphite url: %s', self.options.graphite_url)
context = ContextHelper()
session = self.context.get_config().get_session()
with context.get_rest(url=self.options.graphite_url) as rstifc:
payload = AttrDict()
payload.what = 'Test run started!'
payload.tags = "testrun start"
payload.data = "Run logs: %s" % session.get_url()
rstifc.api.post('/events/', payload=payload)
def now_as_timestamp(self):
now = datetime.datetime.now()
return int(time.mktime(now.timetuple()) * 1e3 + now.microsecond / 1e3)
def graphite_stop(self, result):
context = ContextHelper()
self.options.stop_time = self.now_as_timestamp()
session = self.context.get_config().get_session()
result_text = "Total: %d, Fail: %d" % \
(result.testsRun - result.notFailCount(),
result.failCount())
with context.get_rest(url=self.options.graphite_url) as rstifc:
payload = AttrDict()
payload.what = 'Test run stopped!'
payload.tags = "testrun stop"
payload.data = "Run logs: %s %s" % (session.get_url(), result_text)
rstifc.api.post('/events/', payload=payload)
def influxdb_start(self):
value = "run started"
session = self.context.get_config().get_session()
self.options.start_time = self.now_as_timestamp()
series = []
for dut in expand_devices(self.duts):
with self.context.get_icontrol_rest(device=dut) as rstifc:
info = rstifc.api.get(DeviceInfo.URI)
point = {
"measurement": 'events',
'fields': {
'value': value,
},
'tags': {
"name": "start",
#"host": info.managementAddress,
"harness": session.get_harness_id(),
"host": dut.address,
"machine": info.machineId,
"run": session.session,
"url": session.get_url(),
},
}
series.append(point)
self.client.write_points(series)
def influxdb_stop(self, result):
value = "Total: %d, Fail: %d" % \
(result.testsRun - result.notFailCount(),
result.failCount())
self.options.stop_time = self.now_as_timestamp(
) + 5000 # 5 second padding
session = self.context.get_config().get_session()
series = []
for dut in expand_devices(self.duts):
with self.context.get_icontrol_rest(device=dut) as rstifc:
info = rstifc.api.get(DeviceInfo.URI)
point = {
"measurement": 'events',
'fields': {
'value': value,
'total': result.testsRun - result.notFailCount(),
'failed': result.failCount()
},
'tags': {
"name": "stop",
#"host": info.managementAddress,
"harness": session.get_harness_id(),
"host": dut.address,
"machine": info.machineId,
"run": session.session,
"url": session.get_url(),
},
}
series.append(point)
self.client.write_points(series)
def begin(self):
#self.graphite_start()
self.influxdb_start()
def finalize(self, result):
self.influxdb_stop(result)
class InfluxDBAdapter(object):
def __init__(self, settings=None, database=None):
settings = deepcopy(settings) or {}
settings.setdefault('host', u'localhost')
settings.setdefault('port', u'8086')
settings.setdefault('username', u'root')
settings.setdefault('password', u'root')
settings.setdefault('database', database)
settings.setdefault('use_udp', False)
settings.setdefault('udp_port', u'4444')
settings['port'] = int(settings['port'])
settings['udp_port'] = int(settings['udp_port'])
self.__dict__.update(**settings)
# Bookeeping for all databases having been touched already
self.databases_written_once = Set()
# Knowledge about all databases to be accessed using UDP
# TODO: Refactor to configuration setting
self.udp_databases = [
{'name': 'luftdaten_info', 'port': u'4445'},
]
self.host_uri = u'influxdb://{host}:{port}'.format(**self.__dict__)
log.info(u'Storage target is {uri}', uri=self.host_uri)
self.influx_client = InfluxDBClient(
host=self.host, port=self.port,
username=self.username, password=self.password,
database=self.database,
timeout=10)
# TODO: Hold references to multiple UDP databases using mapping "self.udp_databases".
self.influx_client_udp = None
if settings['use_udp']:
self.influx_client_udp = InfluxDBClient(
host=self.host, port=self.port,
username=self.username, password=self.password,
use_udp=settings['use_udp'], udp_port=settings['udp_port'],
timeout=10)
def is_udp_database(self, name):
for entry in self.udp_databases:
if entry['name'] == name:
return True
return False
def write(self, meta, data):
meta_copy = deepcopy(dict(meta))
data_copy = deepcopy(data)
try:
chunk = self.format_chunk(meta, data)
except Exception as ex:
log.failure(u'Could not format chunk (ex={ex_name}: {ex}): data={data}, meta={meta}',
ex_name=ex.__class__.__name__, ex=ex, meta=meta_copy, data=data_copy)
raise
try:
success = self.write_chunk(meta, chunk)
return success
except requests.exceptions.ConnectionError as ex:
log.failure(u'Problem connecting to InfluxDB at {uri}: {ex}', uri=self.host_uri, ex=ex)
raise
except InfluxDBClientError as ex:
if ex.code == 404 or ex.message == 'database not found':
log.info('Creating database "{database}"', database=meta.database)
self.influx_client.create_database(meta.database)
# Attempt second write
success = self.write_chunk(meta, chunk)
return success
#log.failure('InfluxDBClientError: {ex}', ex=ex)
# [0.8] ignore "409: database kotori-dev exists"
# [0.9] ignore "database already exists"
elif ex.code == 409 or ex.message == 'database already exists':
pass
else:
raise
def write_chunk(self, meta, chunk):
if self.influx_client_udp and self.is_udp_database(meta.database) and meta.database in self.databases_written_once:
success = self.influx_client_udp.write_points([chunk], time_precision='s', database=meta.database)
else:
success = self.influx_client.write_points([chunk], time_precision=chunk['time_precision'], database=meta.database)
self.databases_written_once.add(meta.database)
if success:
log.debug(u"Storage success: {chunk}", chunk=chunk)
else:
log.error(u"Storage failed: {chunk}", chunk=chunk)
return success
@staticmethod
def get_tags(data):
return project(data, ['gateway', 'node'])
def format_chunk(self, meta, data):
"""
Format for InfluxDB >= 0.9::
{
"measurement": "hiveeyes_100",
"tags": {
"host": "server01",
"region": "europe"
},
"time": "2015-10-17T19:30:00Z",
"fields": {
"value": 0.42
}
}
"""
assert isinstance(data, dict), 'Data payload is not a dictionary'
chunk = {
"measurement": meta['measurement'],
"tags": {},
}
"""
if "gateway" in meta:
chunk["tags"]["gateway"] = meta["gateway"]
if "node" in meta:
chunk["tags"]["node"] = meta["node"]
"""
# Extract timestamp field from data
chunk['time_precision'] = 'n'
for time_field in ['time', 'datetime', 'dateTime']:
if time_field in data:
# WeeWX. TODO: Move to specific vendor configuration.
# Disabled in favor of precision detection heuristic.
#if time_field == 'dateTime':
# chunk['time_precision'] = 's'
# Process timestamp field.
if data[time_field]:
# Decode timestamp.
chunk['time'] = data[time_field]
if is_number(chunk['time']):
chunk['time'] = int(float(chunk['time']))
# Remove timestamp from data payload.
del data[time_field]
# If we found a timestamp field already,
# don't look out for more.
break
# Extract geohash from data. Finally, thanks Rich!
# TODO: Also precompute geohash with 3-4 different zoomlevels and add them as tags
if "geohash" in data:
chunk["tags"]["geohash"] = data["geohash"]
del data['geohash']
# Extract more information specific to luftdaten.info
for field in ['location', 'location_id', 'location_name', 'sensor_id', 'sensor_type']:
if field in data:
chunk["tags"][field] = data[field]
del data[field]
# TODO: Maybe do this at data acquisition / transformation time, not here.
if 'time' in chunk:
timestamp = chunk['time'] = parse_timestamp(chunk['time'])
# Heuristically compute timestamp precision
if isinstance(timestamp, int):
if timestamp >= 1e17 or timestamp <= -1e17:
time_precision = 'n'
elif timestamp >= 1e14 or timestamp <= -1e14:
time_precision = 'u'
elif timestamp >= 1e11 or timestamp <= -1e11:
time_precision = 'ms'
# FIXME: Is this a reasonable default?
else:
time_precision = 's'
chunk['time_precision'] = time_precision
"""
# FIXME: Breaks CSV data acquisition. Why?
if isinstance(chunk['time'], datetime.datetime):
if chunk['time'].microsecond == 0:
chunk['time_precision'] = 's'
"""
"""
Prevent errors like
ERROR: InfluxDBClientError: 400:
write failed: field type conflict:
input field "pitch" on measurement "01_position" is type float64, already exists as type integer
"""
self.data_to_float(data)
assert data, 'Data payload is empty'
chunk["fields"] = data
return chunk
def data_to_float(self, data):
return convert_floats(data)
for key, value in data.iteritems():
# Sanity checks
if type(value) in types.StringTypes:
continue
if value is None:
data[key] = None
continue
# Convert to float
try:
data[key] = float(value)
except (TypeError, ValueError) as ex:
log.warn(u'Measurement "{key}: {value}" float conversion failed: {ex}', key=key, value=value, ex=ex)
def update_data(client: InfluxDBClient,
measurement: str,
tag_info: dict,
time: str,
fields: dict,
keep=True):
if keep == False:
raise NotImplementedError
sql_query = (f"SELECT *::field FROM {measurement} "
f"WHERE time='{time}' ")
for key, value in tag_info.items():
sql_query = sql_query + f" and {key}='{value}'"
# print(f"sql_query = {sql_query}")
results = client.query(sql_query)
# print(f"results: {results}")
points = results.get_points()
# print(f"points: {points}")
size = 0
for _point in points:
size += 1
if size == 1:
point = _point
if size == 0:
print("sql query returns empty result set")
raise Exception
# print(f"point = {point}")
''' update point
point sample:
point = {
'time': '2021-01-23T15:58:30.750000Z',
'acceleration': 0.005427043900637734,
'gyroscope': 0.009184479105195894,
'latitude': 31.143920631980222,
'longitude': 121.36646195555223,
'velocity': 0.0
}
'''
updated_point = {
"measurement": measurement,
"time": time,
"tags": tag_info,
}
updated_fields = {}
# update fields for old point
for key, value in point.items():
if key == 'time':
continue
if key in fields.keys():
updated_fields[key] = fields[key]
else:
updated_fields[key] = value
# add fields who does not exist in old point
for key, value in fields.items():
if key not in point.keys():
updated_fields[key] = value
updated_point["fields"] = updated_fields
body = [updated_point]
# print(body)
client.write_points(body)
class InfluxDBMetrics(BaseMetrics):
"""
InfluxDB metrics implementation.
For use of :class:`InfluxDBMetrics` write into :mod:`settings`:
.. code-block:: python
METRICS = {
'backend': 'jobslib.metrics.influxdb.InfluxDBMetrics',
'options': {
'host': 'hostname',
'port': 8086,
'username': '******',
'password': '******',
'database': 'dbname',
},
}
Or use
:envvar:`JOBSLIB_METRICS_INFLUXDB_HOST`,
:envvar:`JOBSLIB_METRICS_INFLUXDB_PORT`,
:envvar:`JOBSLIB_METRICS_INFLUXDB_USERNAME`,
:envvar:`JOBSLIB_METRICS_INFLUXDB_PASSWORD` and
:envvar:`JOBSLIB_METRICS_INFLUXDB_DBNAME` environment variables.
"""
class OptionsConfig(ConfigGroup):
"""
Consul liveness options.
"""
@option(required=True, attrtype=str)
def host(self):
"""
InfluxDB host
"""
host = os.environ.get('JOBSLIB_METRICS_INFLUXDB_HOST')
if host:
return host
return self._settings.get('host', 'localhost')
@option(attrtype=int)
def port(self):
"""
InfluxDB port
"""
port = os.environ.get('JOBSLIB_METRICS_INFLUXDB_PORT')
if port:
return int(port)
return self._settings.get('port', 8086)
@option(required=True, attrtype=str)
def username(self):
"""
InfluxDB username
"""
username = os.environ.get('JOBSLIB_METRICS_INFLUXDB_USERNAME')
if username:
return username
return self._settings.get('username', 'root')
@option(required=True, attrtype=str)
def password(self):
"""
InfluxDB password
"""
password = os.environ.get('JOBSLIB_METRICS_INFLUXDB_PASSWORD')
if password:
return password
return self._settings.get('password', 'root')
@option(attrtype=str)
def database(self):
"""
InfluxDB database
"""
database = os.environ.get('JOBSLIB_METRICS_INFLUXDB_DBNAME')
if database:
return database
return self._settings['database']
def __init__(self, context, options):
super().__init__(context, options)
self._influxdb = InfluxDBClient(
host=self.options.host,
port=self.options.port,
username=self.options.username,
password=self.options.password,
database=self.options.database,
)
def push(self, metrics):
current_dt = datetime.datetime.utcfromtimestamp(time.time())
ts = current_dt.strftime('%Y-%m-%dT%H:%M:%SZ')
task_name = self.context.config.task_class.name
try:
points = []
for metric_name, metric_value in metrics.items():
tags = {
'task': task_name,
}
for k, v in metric_value.get('tags', {}).items():
if k in tags:
raise Exception("Tag '{}' is reserved".format(k))
tags[k] = v
metric = {
'measurement': metric_name,
'tags': tags,
'time': ts,
'fields': {
'value': float(metric_value['value']),
},
}
points.append(metric)
self._influxdb.write_points(points)
except Exception:
logger.exception('Push monitoring metrics into InfluxDb failed')
from config import *
influx_reader = DataFrameClient(
host="192.168.0.4",
port=INFLUXDB_PORT,
username=INFLUXDB_USER,
password=INFLUXDB_PASS,
database="rtlamr",
)
# set to test mode. when ready, change to actual destination
# and uncomment influx_writer.write line below
influx_writer = InfluxDBClient(
host="192.168.0.4",
port=INFLUXDB_PORT,
username=INFLUXDB_USER,
password=INFLUXDB_PASS,
database="rtlamr_test",
)
def copy_meters_data(meters, format_type, start_time, end_time):
"""copy data for given meters from secondary to primary
Args:
meters (list[int]): list of meter ids
format_type (str): lineformat type to use - must be 'SCM' or 'R900'
start_time (datetime): start time of data window to copy
end_time (datetime): end time of data window to copy
"""
try:
from urllib.parse import parse_qs
except ImportError:
from urlparse import parse_qs
# build the response gif
gif = base64.b64decode("R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==")
# init influxdb client
host = os.environ.get("INFLUXER_INFLUXDB_HOST_IP", "localhost")
port = os.environ.get("INFLUXER_INFLUXDB_PORT", 8086)
user = os.environ.get("INFLUXER_INFLUXDB_USERNAME", "root")
pwd = os.environ.get("INFLUXER_INFLUXDB_PASSWORD", "root")
db = os.environ.get("INFLUXER_INFLUXDB_DB", "influxdb")
influx_client = InfluxDBClient(host, port, user, pwd, db)
# init the gevent queue
events_queue = queue.Queue()
flush_interval = os.environ.get("INFLUXER_FLUSH_INTERVAL", 60) # seconds
# acceptable site names
site_names = ("onion", "avclub", "clickhole", "onionstudios", "onionads", )
# init the logger
logger = logging.getLogger('influxer')
logger.setLevel(logging.INFO)
# main wait loop
def count_events():
def init(self, **params):
self.influx_client = InfluxDBClient(**params)
supermann.utils.getLogger(self).info(
"Using InfluxOutput with params %s", str(params))
hbdata = SingleFigure("heartbeats")
hbshift = None
for row in hbfile:
if (row[0] != "time"):
if (hbshift == None):
hbshift = long(row[0][:-6])
if (len(hbdata.x) > 0):
hbdata.add(
float(long(row[0][:-6]) - hbshift) / 1000.,
hbdata.y[len(hbdata.y) - 1])
hbdata.add(float(long(row[0][:-6]) - hbshift) / 1000., int(0))
hbdata.add(float(long(row[0][:-6]) - hbshift) / 1000., int(row[1]))
# grab the data from influxdb
influxclient = InfluxDBClient(host=influxhost,
port=influxport,
database=influxdb)
groupings = {"other.other": []}
for iserie in influxclient.get_list_series(influxdb):
for ilist in iserie["tags"]:
if (not ilist["key"].startswith("cpu.trace")):
tags = split(split(ilist["key"], sep=",")[0], sep=".")
if (len(tags) >= 2):
if (tags[0] + "." + tags[1]) in groupings:
groupings[tags[0] + "." + tags[1]].append(ilist["key"])
else:
groupings[tags[0] + "." + tags[1]] = [ilist["key"]]
else:
groupings["other.other"].append(ilist["key"])
for igroupname in groupings:
figs = []
class InfluxDBAdapter(object):
def __init__(self, settings=None, database=None):
settings = deepcopy(settings) or {}
settings.setdefault('host', u'localhost')
settings.setdefault('port', u'8086')
settings.setdefault('username', u'root')
settings.setdefault('password', u'root')
settings.setdefault('database', database)
settings.setdefault('use_udp', False)
settings.setdefault('udp_port', u'4444')
settings['port'] = int(settings['port'])
settings['udp_port'] = int(settings['udp_port'])
self.__dict__.update(**settings)
# Bookeeping for all databases having been touched already
self.databases_written_once = Set()
# Knowledge about all databases to be accessed using UDP
# TODO: Refactor to configuration setting
self.udp_databases = [
{
'name': 'luftdaten_info',
'port': u'4445'
},
]
self.host_uri = u'influxdb://{host}:{port}'.format(**self.__dict__)
log.info(u'Storage target is {uri}', uri=self.host_uri)
self.influx_client = InfluxDBClient(host=self.host,
port=self.port,
username=self.username,
password=self.password,
database=self.database,
timeout=10)
# TODO: Hold references to multiple UDP databases using mapping "self.udp_databases".
self.influx_client_udp = None
if settings['use_udp']:
self.influx_client_udp = InfluxDBClient(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
use_udp=settings['use_udp'],
udp_port=settings['udp_port'],
timeout=10)
def is_udp_database(self, name):
for entry in self.udp_databases:
if entry['name'] == name:
return True
return False
def write(self, meta, data):
meta_copy = deepcopy(dict(meta))
data_copy = deepcopy(data)
try:
chunk = self.format_chunk(meta, data)
except Exception as ex:
log.failure(
u'Could not format chunk (ex={ex_name}: {ex}): data={data}, meta={meta}',
ex_name=ex.__class__.__name__,
ex=ex,
meta=meta_copy,
data=data_copy)
raise
try:
success = self.write_chunk(meta, chunk)
return success
except requests.exceptions.ConnectionError as ex:
log.failure(u'Problem connecting to InfluxDB at {uri}: {ex}',
uri=self.host_uri,
ex=ex)
raise
except InfluxDBClientError as ex:
if ex.code == 404 or ex.message == 'database not found':
log.info('Creating database "{database}"',
database=meta.database)
self.influx_client.create_database(meta.database)
# Attempt second write
success = self.write_chunk(meta, chunk)
return success
#log.failure('InfluxDBClientError: {ex}', ex=ex)
# [0.8] ignore "409: database kotori-dev exists"
# [0.9] ignore "database already exists"
elif ex.code == 409 or ex.message == 'database already exists':
pass
else:
raise
def write_chunk(self, meta, chunk):
if self.influx_client_udp and self.is_udp_database(
meta.database
) and meta.database in self.databases_written_once:
success = self.influx_client_udp.write_points(
[chunk], time_precision='s', database=meta.database)
else:
success = self.influx_client.write_points(
[chunk],
time_precision=chunk['time_precision'],
database=meta.database)
self.databases_written_once.add(meta.database)
if success:
log.debug(u"Storage success: {chunk}", chunk=chunk)
else:
log.error(u"Storage failed: {chunk}", chunk=chunk)
return success
@staticmethod
def get_tags(data):
return project(data, ['gateway', 'node'])
def format_chunk(self, meta, data):
"""
Format for InfluxDB >= 0.9::
{
"measurement": "hiveeyes_100",
"tags": {
"host": "server01",
"region": "europe"
},
"time": "2015-10-17T19:30:00Z",
"fields": {
"value": 0.42
}
}
"""
assert isinstance(data, dict), 'Data payload is not a dictionary'
chunk = {
"measurement": meta['measurement'],
"tags": {},
}
"""
if "gateway" in meta:
chunk["tags"]["gateway"] = meta["gateway"]
if "node" in meta:
chunk["tags"]["node"] = meta["node"]
"""
# Extract timestamp field from data
chunk['time_precision'] = 'n'
for time_field in ['time', 'datetime', 'dateTime']:
if time_field in data:
# WeeWX. TODO: Move to specific vendor configuration.
# Disabled in favor of precision detection heuristic.
#if time_field == 'dateTime':
# chunk['time_precision'] = 's'
# Process timestamp field.
if data[time_field]:
# Decode timestamp.
chunk['time'] = data[time_field]
if is_number(chunk['time']):
chunk['time'] = int(float(chunk['time']))
# Remove timestamp from data payload.
del data[time_field]
# If we found a timestamp field already,
# don't look out for more.
break
# Extract geohash from data. Finally, thanks Rich!
# TODO: Also precompute geohash with 3-4 different zoomlevels and add them as tags
if "geohash" in data:
chunk["tags"]["geohash"] = data["geohash"]
del data['geohash']
if "latitude" in data and "longitude" in data:
chunk["tags"]["latitude"] = data["latitude"]
chunk["tags"]["longitude"] = data["longitude"]
del data['latitude']
del data['longitude']
# Extract more information specific to luftdaten.info
for field in [
'location', 'location_id', 'location_name', 'sensor_id',
'sensor_type'
]:
if field in data:
chunk["tags"][field] = data[field]
del data[field]
# TODO: Maybe do this at data acquisition / transformation time, not here.
if 'time' in chunk:
timestamp = chunk['time'] = parse_timestamp(chunk['time'])
# Heuristically compute timestamp precision
if isinstance(timestamp, int):
if timestamp >= 1e17 or timestamp <= -1e17:
time_precision = 'n'
elif timestamp >= 1e14 or timestamp <= -1e14:
time_precision = 'u'
elif timestamp >= 1e11 or timestamp <= -1e11:
time_precision = 'ms'
# FIXME: Is this a reasonable default?
else:
time_precision = 's'
chunk['time_precision'] = time_precision
"""
# FIXME: Breaks CSV data acquisition. Why?
if isinstance(chunk['time'], datetime.datetime):
if chunk['time'].microsecond == 0:
chunk['time_precision'] = 's'
"""
"""
Prevent errors like
ERROR: InfluxDBClientError: 400:
write failed: field type conflict:
input field "pitch" on measurement "01_position" is type float64, already exists as type integer
"""
self.data_to_float(data)
assert data, 'Data payload is empty'
chunk["fields"] = data
return chunk
def data_to_float(self, data):
return convert_floats(data)
for key, value in data.iteritems():
# Sanity checks
if type(value) in types.StringTypes:
continue
if value is None:
data[key] = None
continue
# Convert to float
try:
data[key] = float(value)
except (TypeError, ValueError) as ex:
log.warn(
u'Measurement "{key}: {value}" float conversion failed: {ex}',
key=key,
value=value,
ex=ex)
def __init__(self):
load_dotenv()
user=os.getenv("INFLUX_DB_USER")
password=os.getenv("INFLUX_DB_PASSWORD")
self.client = InfluxDBClient('localhost', 8086, user, password, 'pi_influxdb')
