def write_to_influxdb(_url, _token, _org, _bucket, _host, _seconds):
"""
Write the response time to Influxdb
:param _url: Influxdb server URL as string
:param _token: Influxdb auth token as string
:param _org: Influxdb organization as string
:param _bucket: Influxdb bucket as string
:param _host: Name of the host being as string
:param _seconds: Response time in seconds as float
:return: None
"""
point = Point("response_time").tag("host",
_host).field("seconds", _seconds)
try:
client = InfluxDBClient(url=_url,
token=_token,
org=_org,
verify_ssl=False)
write_api = client.write_api(write_options=SYNCHRONOUS)
write_api.write(bucket=_bucket, record=point)
client.close()
except Exception as e:
print("CRITICAL - Failed to write to influxdb: %s" % e)
sys.exit(2)
return None
def main():
parse_row.progress = 0
url = "https://github.com/influxdata/influxdb-client-python/wiki/data/stock-prices-example.csv"
response = requests.get(url, stream=True)
data = rx \
.from_iterable(DictReader(response.iter_lines(decode_unicode=True))) \
.pipe(ops.map(lambda row: parse_row(row)))
client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
debug=False)
write_api = client.write_api(
write_options=WriteOptions(batch_size=50_000, flush_interval=10_000))
write_api.write(bucket="my-bucket", record=data)
write_api.close()
query = '''
from(bucket:"my-bucket")
|> range(start: 0, stop: now())
|> filter(fn: (r) => r._measurement == "financial-analysis")
|> filter(fn: (r) => r.symbol == "AAPL")
|> filter(fn: (r) => r._field == "close")
|> drop(columns: ["_start", "_stop", "table", "_field","_measurement"])
'''
result = client.query_api().query_data_frame(query=query)
print(result.head(100))
"""
Close client
"""
client.close()
class InfluxDBDataWriter(CovidDataWriter):
def __init__(self, url: str, token: str, org: str, bucket: str):
self._org = org
self._bucket = bucket
self._client = InfluxDBClient(url=url, token=token)
self._write_api = self._client.write_api()
def write_data(self, data_point: CovidDataPoint) -> None:
self._write_api.write(
bucket=self._bucket,
org=self._org,
record=self._convert_datapoint(data_point=data_point),
)
@staticmethod
def _convert_datapoint(data_point: CovidDataPoint) -> Point:
point = (Point(data_point.measurement.value).tag(
"iso3", data_point.iso_3).tag("country", data_point.country).tag(
"state_province", data_point.state_province).tag(
"county",
data_point.county).tag("lat", data_point.lat).tag(
"long",
data_point.long).field("count", data_point.value).time(
data_point.timestamp))
return point
def __del__(self):
self._client.close()
class InfluxDBWriter(multiprocessing.Process):
"""
Writer that writes data in batches with 50_000 items.
"""
def __init__(self, queue):
multiprocessing.Process.__init__(self)
self.queue = queue
self.client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
debug=False)
self.write_api = self.client.write_api(
write_options=WriteOptions(write_type=WriteType.batching,
batch_size=50_000,
flush_interval=10_000))
def run(self):
while True:
next_task = self.queue.get()
if next_task is None:
# Poison pill means terminate
self.terminate()
self.queue.task_done()
break
self.write_api.write(bucket="my-bucket", record=next_task)
self.queue.task_done()
def terminate(self) -> None:
proc_name = self.name
print()
print('Writer: flushing data...')
self.write_api.close()
self.client.close()
print('Writer: closed'.format(proc_name))
def send_data(what, value):
"""
this function sends the data as json, and checks if it was succesfull
"""
client = InfluxDBClient(url=influx_cloud_url, token=influx_cloud_token)
try:
point = Point("measurement").field(what,
value).time(time=datetime.utcnow())
# if data is incorrect, don't send anything
if point is None:
return -1
"""
Write data by Point structure
"""
print(f'Writing to InfluxDB cloud: {point.to_line_protocol()} ...')
write_api = client.write_api(write_options=SYNCHRONOUS)
write_api.write(bucket=bucket, org=org, record=point)
print()
print('success')
print()
print()
except Exception as e:
print(e)
finally:
client.close()
class TestHealthMock(unittest.TestCase):
def setUp(self) -> None:
httpretty.enable()
httpretty.reset()
self.influxdb_client = InfluxDBClient(url="http://localhost",
token="my-token")
def tearDown(self) -> None:
self.influxdb_client.close()
httpretty.disable()
def test_without_retry(self):
httpretty.register_uri(httpretty.GET,
uri="http://localhost/health",
status=429,
adding_headers={
'Retry-After': '5',
'Content-Type': 'application/json'
},
body="{\"message\":\"Health is not working\"}")
check = self.influxdb_client.health()
self.assertTrue("Health is not working" in check.message,
msg=check.message)
self.assertEqual(check.status, "fail")
self.assertEqual(check.name, "influxdb")
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
def test_with_retry(self):
self.influxdb_client.close()
self.influxdb_client = InfluxDBClient(url="http://localhost",
token="my-token",
retries=Retry())
httpretty.register_uri(
httpretty.GET,
uri="http://localhost/health",
status=200,
adding_headers={'Content-Type': 'application/json'},
body=
"{\"message\":\"ready for queries and writes\", \"name\":\"influxdb\", \"status\":\"pass\"}"
)
httpretty.register_uri(httpretty.GET,
uri="http://localhost/health",
status=429,
adding_headers={
'Retry-After': '1',
'Content-Type': 'application/json'
},
body="{\"message\":\"Health is not working\"}")
health = self.influxdb_client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
self.assertEqual(2, len(httpretty.httpretty.latest_requests))
class BaseTest(unittest.TestCase):
def setUp(self) -> None:
self.conf = influxdb_client.configuration.Configuration()
self.host = os.getenv('INFLUXDB_V2_URL', "http://localhost:8086")
self.debug = False
self.auth_token = os.getenv('INFLUXDB_V2_TOKEN', "my-token")
self.org = os.getenv('INFLUXDB_V2_ORG', "my-org")
self.client = InfluxDBClient(url=self.host,
token=self.auth_token,
debug=self.debug,
org=self.org)
self.api_client = self.client.api_client
self.query_api = self.client.query_api()
self.buckets_api = self.client.buckets_api()
self.users_api = self.client.users_api()
self.organizations_api = self.client.organizations_api()
self.authorizations_api = self.client.authorizations_api()
self.labels_api = self.client.labels_api()
self.my_organization = self.find_my_org()
def tearDown(self) -> None:
self.client.close()
def create_test_bucket(self):
bucket_name = generate_bucket_name()
bucket = self.buckets_api.create_bucket(bucket_name=bucket_name,
org=self.my_organization,
description=bucket_name +
"description")
return bucket
def delete_test_bucket(self, bucket):
return self.buckets_api.delete_bucket(bucket)
def find_my_org(self) -> Organization:
return self.client.organizations_api().find_organizations(
org=self.org)[0]
@staticmethod
def log(args):
print(">>>", args)
@staticmethod
def generate_name(prefix):
assert prefix != "" or prefix is not None
return prefix + str(datetime.datetime.now().timestamp()) + "-IT"
@classmethod
def retention_rule(cls) -> BucketRetentionRules:
return BucketRetentionRules(type='expire', every_seconds=3600)
def assertEqualIgnoringWhitespace(self, first, second, msg=None) -> None:
whitespace_pattern = re.compile(r"\s+")
self.assertEqual(whitespace_pattern.sub("", first),
whitespace_pattern.sub("", second),
msg=msg)
def test_ping_not_running_instance(self):
client_not_running = InfluxDBClient("http://localhost:8099",
token="my-token",
debug=True)
ping = client_not_running.ping()
self.assertFalse(ping)
client_not_running.close()
def test_version_not_running_instance(self):
client_not_running = InfluxDBClient("http://localhost:8099",
token="my-token",
debug=True)
with self.assertRaises(NewConnectionError):
client_not_running.version()
client_not_running.close()
def write_data(self, data: [Point], bucket):
client = InfluxDBClient(url="http://%s:%s" % (self.host, self.port),
token=self.token,
org=self.org,
timeout=1000)
_write_client = client.write_api(write_options=SYNCHRONOUS)
_write_client.write(bucket=bucket, org=self.org, record=data)
_write_client.close()
client.close()
def on_exit(db_client: InfluxDBClient, write_api: WriteApi):
"""Close clients after terminate a script.
:param db_client: InfluxDB client
:param write_api: WriteApi
:return: nothing
"""
write_api.close()
db_client.close()
def send_to_influxdb():
client = InfluxDBClient(
url="https://eu-central-1-1.aws.cloud2.influxdata.com", token=my_token)
try:
kind = 'temperature'
device = 'opt-123'
"""
Get sensor data
"""
# pressure = sense.get_pressure()
pressure = 339.3
# pressure = round(pressure, 1)
temp = 12.0
# temp = sense.get_temperature()
# temp = round(temp, 1)
# humid = sense.get_humidity()
humid = 99
"""
Write data by Point structure
"""
point = Point(kind).tag('owner', owner).tag('device', device).field(
'value', round(outside_temp(), 2)).time(time=datetime.utcnow())
print(f'Writing to InfluxDB cloud: {point.to_line_protocol()} ...')
write_api = client.write_api(write_options=SYNCHRONOUS)
write_api.write(bucket=bucket, org=org, record=point)
print()
print('success')
print()
print()
if False:
"""
Query written data
"""
query = f'from(bucket: "{bucket}") |> range(start: -1d) |> filter(fn: (r) => r._measurement == "{kind}")'
print(f'Querying from InfluxDB cloud: "{query}" ...')
print()
query_api = client.query_api()
tables = query_api.query(query=query, org=org)
for table in tables:
for row in table.records:
print(
f'{row.values["_time"]}: owner={row.values["owner"]},device={row.values["device"]} '
f'{row.values["_value"]} °C')
print()
print('success')
except Exception as e:
print(e)
finally:
client.close()
return
class HelpersTest(BaseTest):
def test_is_id(self):
self.assertTrue(_is_id("ffffffffffffffff"))
self.assertTrue(_is_id("020f755c3c082000"))
self.assertTrue(_is_id("ca55e77eca55e77e"))
self.assertTrue(_is_id("02def021097c6000"))
self.assertFalse(_is_id("gggggggggggggggg"))
self.assertFalse(_is_id("abc"))
self.assertFalse(_is_id("abcdabcdabcdabcd0"))
self.assertFalse(_is_id("020f75"))
self.assertFalse(_is_id("020f755c3c082000aaa"))
self.assertFalse(_is_id(None))
def test_organization_as_query_param(self):
organization = Organization(id="org-id", name="org-name")
org = get_org_query_param(organization, self.client)
self.assertEqual("org-id", org)
def test_required_id(self):
org = get_org_query_param(None, self.client, required_id=True)
self.assertEqual(self.my_organization.id, org)
def test_required_id_not_exist(self):
with pytest.raises(InfluxDBError) as e:
get_org_query_param("not_exist_name",
self.client,
required_id=True)
assert "The client cannot find organization with name: 'not_exist_name' to determine their ID." in f"{e.value} "
def test_both_none(self):
self.client.close()
self.client = InfluxDBClient(url=self.client.url, token="my-token")
org = get_org_query_param(None, self.client)
self.assertIsNone(org)
def test_not_permission_to_read_org(self):
# Create Token without permission to read Organizations
resource = PermissionResource(type="buckets",
org_id=self.find_my_org().id)
authorization = self.client \
.authorizations_api() \
.create_authorization(org_id=self.find_my_org().id,
permissions=[Permission(resource=resource, action="read"),
Permission(resource=resource, action="write")])
self.client.close()
# Initialize client without permission to read Organizations
self.client = InfluxDBClient(url=self.client.url,
token=authorization.token)
with pytest.raises(InfluxDBError) as e:
get_org_query_param("my-org", self.client, required_id=True)
assert "The client cannot find organization with name: 'my-org' to determine their ID. Are you using token " \
"with sufficient permission?" in f"{e.value} "
def test_check_write_permission_by_empty_data(self):
client = InfluxDBClient(url="http://localhost:8086", token="my-token-wrong", org="my-org")
write_api = client.write_api(write_options=SYNCHRONOUS)
with self.assertRaises(ApiException) as cm:
write_api.write("my-bucket", self.org, b'')
exception = cm.exception
self.assertEqual(401, exception.status)
self.assertEqual("Unauthorized", exception.reason)
client.close()
def main():
# Main logic resides here.
# Connect to local influx db container.
config_json = load_config()
client = InfluxDBClient(url='http://localhost:8086',
token=config_json['token'],
org=config_json['org'])
try:
while True:
is_chrome_open = False
list_procs = ""
output = subprocess.Popen(["ps", "aux"], stdout=subprocess.PIPE)
list_procs = output.stdout.read()
if 'Google Chrome'.encode('utf-8') in list_procs:
is_chrome_open = True
if is_chrome_open:
all_tabs = subprocess.Popen(
["osascript", "show_tabs_chrome.scpt"],
stdout=subprocess.PIPE)
all_tabs = all_tabs.stdout.read()
all_tabs = all_tabs.decode('utf-8').split('\n')
parsed_links = set()
parsed_active_link = ''
for link in all_tabs:
if link:
string_form = link
if string_form.split()[0] != 'Active:':
parsed_url = parse_url(string_form.split()[0])
parsed_links.add(parsed_url)
elif string_form.split()[0] == 'Active:':
parsed_active_link = parse_url(
string_form.split()[1])
write_api = client.write_api(write_options=SYNCHRONOUS)
query_api = client.query_api()
for link in parsed_links:
active_time = 0 if parsed_active_link != link else 1
inactive_time = 1 if parsed_active_link != link else 0
insert_link_query(write_api, link,
config_json['bucket_name'], active_time,
inactive_time)
time.sleep(1)
except Exception as e:
client.close()
def test_query_without_credentials(self):
_client = InfluxDBClient(url="http://localhost:9999", token="my-token-wrong-credentials", org="my-org")
with self.assertRaises(ApiException) as ae:
query = 'from(bucket: "my-bucket")' \
'|> range(start: 2020-02-19T23:30:00Z, stop: now())' \
f'|> filter(fn: (r) => r._measurement == "my-measurement")'
_client.query_api().query_data_frame(query=query)
exception = ae.exception
self.assertEqual(401, exception.status)
self.assertEqual("Unauthorized", exception.reason)
_client.close()
def refreshLatencyGraphs(secondsToRun):
startTime = datetime.now()
with open('statsByParentNode.json', 'r') as j:
parentNodes = json.loads(j.read())
with open('statsByDevice.json', 'r') as j:
devices = json.loads(j.read())
print("Retrieving device statistics")
devices = getLatencies(devices, secondsToRun)
print("Computing parent node statistics")
parentNodes = getParentNodeStats(parentNodes, devices)
print("Writing data to InfluxDB")
bucket = influxDBBucket
org = influxDBOrg
token = influxDBtoken
url="http://localhost:8086"
client = InfluxDBClient(
url=url,
token=token,
org=org
)
write_api = client.write_api(write_options=SYNCHRONOUS)
queriesToSend = []
for device in devices:
if device['tcpLatency'] != None:
p = Point('Latency').tag("Device", device['hostname']).tag("ParentNode", device['ParentNode']).tag("Type", "Device").field("TCP Latency", device['tcpLatency'])
queriesToSend.append(p)
for parentNode in parentNodes:
if parentNode['tcpLatency'] != None:
p = Point('Latency').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("TCP Latency", parentNode['tcpLatency'])
queriesToSend.append(p)
write_api.write(bucket=bucket, record=queriesToSend)
print("Added " + str(len(queriesToSend)) + " points to InfluxDB.")
client.close()
#with open('statsByParentNode.json', 'w') as infile:
# json.dump(parentNodes, infile)
#with open('statsByDevice.json', 'w') as infile:
# json.dump(devices, infile)
endTime = datetime.now()
durationSeconds = round((endTime - startTime).total_seconds())
print("Graphs updated within " + str(durationSeconds) + " seconds.")
class InfluxDBWriterToPickle:
def __init__(self):
self.client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
debug=False)
self.write_api = self.client.write_api(
write_options=WriteOptions(write_type=WriteType.batching,
batch_size=50_000,
flush_interval=10_000))
def write(self, record):
self.write_api.write(bucket="my-bucket", record=record)
def terminate(self) -> None:
self.write_api.close()
self.client.close()
class WriteApiTestMock(BaseTest):
def setUp(self) -> None:
httpretty.enable()
httpretty.reset()
conf = influxdb_client.configuration.Configuration()
conf.host = "http://localhost"
conf.debug = False
self.influxdb_client = InfluxDBClient(url=conf.host, token="my-token")
def tearDown(self) -> None:
self.influxdb_client.close()
httpretty.disable()
def test_writes_synchronous_without_retry(self):
httpretty.register_uri(httpretty.POST,
uri="http://localhost/api/v2/write",
status=503)
self.write_client = self.influxdb_client.write_api(
write_options=SYNCHRONOUS)
with self.assertRaises(ApiException) as cm:
self.write_client.write(
"my-bucket", "my-org",
"h2o_feet,location=coyote_creek water_level=1 1")
exception = cm.exception
self.assertEqual("Service Unavailable", exception.reason)
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
def test_writes_asynchronous_without_retry(self):
httpretty.register_uri(httpretty.POST,
uri="http://localhost/api/v2/write",
status=503)
self.write_client = self.influxdb_client.write_api(
write_options=ASYNCHRONOUS)
with self.assertRaises(ApiException) as cm:
self.write_client.write(
"my-bucket", "my-org",
"h2o_feet,location=coyote_creek water_level=1 1").get()
exception = cm.exception
self.assertEqual("Service Unavailable", exception.reason)
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
class InfluxDBClientAuthorization(unittest.TestCase):
def setUp(self) -> None:
httpretty.enable()
httpretty.reset()
def tearDown(self) -> None:
if self.influxdb_client:
self.influxdb_client.close()
httpretty.disable()
def test_session_request(self):
httpretty.reset()
self.influxdb_client = InfluxDBClient(url="http://localhost",
token="my-token",
username="******",
password="******")
# create user session
httpretty.register_uri(httpretty.POST,
uri="http://localhost/api/v2/signin",
adding_headers={'Set-Cookie': 'session=xyz'})
# authorized request
httpretty.register_uri(httpretty.GET, uri="http://localhost/ping")
# expires current session
httpretty.register_uri(httpretty.POST,
uri="http://localhost/api/v2/signout")
ping = self.influxdb_client.ping()
self.assertTrue(ping)
self.assertEqual(2, len(httpretty.httpretty.latest_requests))
# basic auth header
self.assertEqual(
'Basic bXktdXNlcm5hbWU6bXktcGFzc3dvcmQ=',
httpretty.httpretty.latest_requests[0].headers['Authorization'])
# cookie header
self.assertEqual(
'session=xyz',
httpretty.httpretty.latest_requests[1].headers['Cookie'])
self.assertIsNotNone(self.influxdb_client.api_client.cookie)
# signout
self.influxdb_client.close()
self.assertEqual(3, len(httpretty.httpretty.latest_requests))
def send(self, vals, batteryAPI):
client = InfluxDBClient(url=self.influx_url,
token=f'{self.influx_user}:{self.influx_pass}',
org='-')
bucket = f'{self.influx_database}/{self.influx_retention_policy}'
write_api = client.write_api()
inverterDetails = vals.copy()
inverterDetails.pop('Serial', None)
point = Point("solax").tag("inverter", vals['name'])
for x, y in inverterDetails.items():
point.field(x, y)
# print(point.to_line_protocol())
# print(inverterDetails)
write_api.write(bucket=bucket, record=point)
write_api.__del__()
client.close()
def ingest_dataframe(df, measurement_name, tag_columns):
"""Write data"""
point_settings = PointSettings()
point_settings.add_default_tag("units", "Wh")
point_settings.add_default_tag("source", "enphase")
points = data_frame_to_list_of_points(
data_frame=df,
point_settings=point_settings,
data_frame_measurement_name=measurement_name,
data_frame_tag_columns=tag_columns)
client = InfluxDBClient(url=url, token=token, org=org)
write_api = client.write_api()
# line protocol: measurementName,tagKey=tagValue fieldKey="fieldValue" timestamp
# write_api.write(bucket=bucket, org=org, record=df,
# data_frame_measurement_name=measurement_name,
# data_frame_tag_columns=tag_columns)
write_api.write(bucket=bucket, org=org, record=points)
write_api.close()
client.close()
class InfluxDB:
def __init__(self, config):
self._config = config
self._client = None
self._write_api = None
self._query_api = None
self._delete_api = None
self._tasks_api = None
self._organizations_api = None
self._token = None
self._org = None
self._url = None
self._bucket = None
def start(self) -> bool:
"""Initialize the InfluxDB client."""
try:
influxdb_options = retrieve_options(self._config, 'influxdb2', _INFLUXDB2_OPTIONS)
debug_options = retrieve_options(self._config, 'debug', _DEBUG_OPTIONS)
except FailedInitialization as e:
_LOGGER.error(f"{e}")
return False
if not influxdb_options.get('enable', None):
_LOGGER.warning("InfluxDB support is disabled in the YAML configuration file")
return True
result = False
try:
self._bucket = influxdb_options.get('bucket', None)
self._url = influxdb_options.get('url', None)
self._token = influxdb_options.get('token', None)
self._org = influxdb_options.get('org', None)
self._client = InfluxDBClient(url=self._url, token=self._token, org=self._org, enable_gzip=True)
if not self._client:
raise FailedInitialization(
f"failed to get InfluxDBClient from '{self._url}' (check url, token, and/or organization)")
self._write_api = self._client.write_api(write_options=SYNCHRONOUS)
self._query_api = self._client.query_api()
self._delete_api = self._client.delete_api()
self._tasks_api = self._client.tasks_api()
self._organizations_api = self._client.organizations_api()
multisma2_debug = os.getenv(_DEBUG_ENV_VAR, 'False').lower() in ('true', '1', 't')
try:
if multisma2_debug and debug_options.get('delete_bucket', False):
self.delete_bucket()
_LOGGER.info(f"Deleted bucket '{self._bucket}' at '{self._url}'")
except InfluxDBBucketError as e:
raise FailedInitialization(f"{e}")
try:
if not self.connect_bucket(multisma2_debug and debug_options.get('create_bucket', False)):
raise FailedInitialization(f"Unable to access (or create) bucket '{self._bucket}' at '{self._url}'")
except InfluxDBBucketError as e:
raise FailedInitialization(f"{e}")
_LOGGER.info(f"Connected to InfluxDB: '{self._url}', bucket '{self._bucket}'")
result = True
except FailedInitialization as e:
_LOGGER.error(f"{e}")
except NewConnectionError:
_LOGGER.error(f"InfluxDB client unable to connect to host at {self._url}")
except ApiException as e:
_LOGGER.error(f"InfluxDB client unable to access bucket '{self._bucket}' at {self._url}: {e.reason}")
except Exception as e:
_LOGGER.error(f"Unexpected exception: {e}")
finally:
return result
def stop(self):
if self._write_api:
self._write_api.close()
self._write_api = None
if self._client:
self._client.close()
self._client = None
def bucket(self):
return self._bucket
def org(self):
return self._org
def write_api(self):
return self._write_api
def query_api(self):
return self._query_api
def delete_api(self):
return self._delete_api
def tasks_api(self):
return self._tasks_api
def organizations_api(self):
return self._organizations_api
def write_points(self, points):
if not self._write_api:
return False
try:
self._write_api.write(bucket=self._bucket, record=points, write_precision=WritePrecision.S)
return True
except ApiException as e:
raise InfluxDBWriteError(f"InfluxDB client unable to write to '{self._bucket}' at {self._url}: {e.reason}")
except Exception as e:
raise InfluxDBWriteError(f"Unexpected failure in write_points(): {e}")
def write_history(self, site, topic):
if not self._write_api:
return False
lookup = LP_LOOKUP.get(topic, None)
if not lookup:
_LOGGER.error(f"write_history(): unknown topic '{topic}'")
return False
measurement = lookup.get('measurement')
tags = lookup.get('tags', None)
field = lookup.get('field', None)
lps = []
for inverter in site:
inverter_name = inverter.pop(0)
name = inverter_name.get('inverter', 'sunnyboy')
for history in inverter:
t = history['t']
v = history['v']
if v is None:
continue
lp = f"{measurement}"
if tags and len(tags):
lp += f",{tags[0]}={name}"
if isinstance(v, int):
lp += f" {field}={v}i {t}"
lps.append(lp)
else:
_LOGGER.error(
f"write_history(): unanticipated type '{type(v)}' in measurement '{measurement}/{field}'")
continue
try:
self._write_api.write(bucket=self._bucket, record=lps, write_precision=WritePrecision.S)
_LOGGER.debug(f"write_history({site}, {topic}): {lps}")
return True
except ApiException as e:
raise InfluxDBWriteError(f"InfluxDB client unable to write to '{self._bucket}' at {self._url}: {e.reason}")
except Exception as e:
raise InfluxDBWriteError(f"Unexpected failure in write_history(): {e}")
def write_sma_sensors(self, sensor, timestamp=None):
if not self._client:
return False
ts = timestamp if timestamp is not None else int(time.time())
lps = []
for old_point in sensor:
point = old_point.copy()
topic = point.pop('topic', None)
point.pop('precision', None)
if topic:
lookup = LP_LOOKUP.get(topic, None)
if not lookup:
_LOGGER.error(f"write_sma_sensors(): unknown topic '{topic}'")
continue
if not lookup.get('output', False):
continue
if topic == 'production/today':
day = datetime.datetime.fromtimestamp(ts).date()
dt = datetime.datetime.combine(day, datetime.time(0, 0))
ts = int(dt.timestamp())
elif topic == 'production/month':
month = datetime.date.fromtimestamp(ts).replace(day=1)
dt = datetime.datetime.combine(month, datetime.time(0, 0))
ts = int(dt.timestamp())
elif topic == 'production/year':
year = datetime.date.fromtimestamp(ts).replace(month=1, day=1)
dt = datetime.datetime.combine(year, datetime.time(0, 0))
ts = int(dt.timestamp())
measurement = lookup.get('measurement')
tags = lookup.get('tags', None)
for k, v in point.items():
field = lookup.get('field')
# sample: dc_measurements
lp = f'{measurement}'
if tags and len(tags):
# sample: dc_measurements,_inverter=sb71
lp += f',{tags[0]}={k}'
if not field:
field = k
if isinstance(v, int):
# sample: ac_measurements,_inverter=sb71 power=0.23 1556813561098
lp += f' {field}={v}i {ts}'
lps.append(lp)
elif isinstance(v, float):
# sample: ac_measurements,_inverter=sb71 power=0.23 1556813561098
lp += f' {field}={v} {ts}'
lps.append(lp)
elif isinstance(v, dict):
lp_prefix = f'{lp}'
for k1, v1 in v.items():
# sample: dc_measurements,_inverter=sb71
lp = f'{lp_prefix}'
if tags and len(tags) > 1:
# sample: dc_measurements,_inverter=sb71,_string=a
lp += f',{tags[1]}={k1}'
if isinstance(v1, int):
# sample: dc_measurements,_inverter=sb71,_string=a power=1000 1556813561098
lp += f' {field}={v1}i {ts}'
lps.append(lp)
elif isinstance(v1, float):
# sample: dc_measurements,_inverter=sb71,_string=a current=0.23 1556813561098
lp += f' {field}={v1} {ts}'
lps.append(lp)
else:
_LOGGER.error(
f"write_sma_sensors(): unanticipated dictionary type '{type(v1)}' in measurement '{measurement}/{field}'")
else:
_LOGGER.error(
f"write_sma_sensors(): unanticipated type '{type(v)}' in measurement '{measurement}/{field}'")
continue
try:
self._write_api.write(bucket=self._bucket, record=lps, write_precision=WritePrecision.S)
return True
except ApiException as e:
raise InfluxDBWriteError(f"InfluxDB client unable to write to '{self._bucket}' at {self._url}: {e.reason}")
except Exception as e:
raise InfluxDBWriteError(f"Unexpected failure in write_sma_sensors(): {e}")
def delete_bucket(self):
if not self._client:
return False
try:
buckets_api = self._client.buckets_api()
found_bucket = buckets_api.find_bucket_by_name(self._bucket)
if found_bucket:
buckets_api.delete_bucket(found_bucket)
bucket = buckets_api.find_bucket_by_name(self._bucket)
if not bucket:
return True
return False
except ApiException as e:
raise InfluxDBBucketError(
f"InfluxDB client unable to delete bucket '{self._bucket}' at {self._url}: {e.reason}")
except Exception as e:
raise InfluxDBBucketError(f"Unexpected exception in delete_bucket(): {e}")
def connect_bucket(self, create_bucket=False):
if not self._client:
return False
try:
buckets_api = self._client.buckets_api()
bucket = buckets_api.find_bucket_by_name(self._bucket)
if bucket:
return True
if create_bucket:
bucket = buckets_api.create_bucket(
bucket_name=self._bucket, org_id=self._org, retention_rules=None, org=None)
if bucket:
_LOGGER.info(f"Created bucket '{self._bucket}' at {self._url}")
return True
return False
except ApiException as e:
raise InfluxDBBucketError(
f"InfluxDB client unable to create bucket '{self._bucket}' at {self._url}: {e.reason}")
except NewConnectionError:
raise
except Exception as e:
raise InfluxDBBucketError(f"Unexpected exception in connect_bucket(): {e}")
class InfluxOutput(Output):
ip: str
port: str
token: str
org: str
bucket: str
measurement: str
url: str
tags: Dict
output_timestamp_name: str
influx_writer: Any
def __init__(self, conf: Dict[Any, Any] = None) -> None:
super().__init__()
if (conf is not None):
self.configure(conf=conf)
def configure(self, conf: Dict[Any, Any] = None) -> None:
super().configure(conf=conf)
# Configura writer
self.ip = conf["ip"]
self.port = conf["port"]
self.token = conf["token"]
self.org = conf["org"]
self.url = "http://" + self.ip + ":" + self.port
self.bucket = conf["bucket"]
self.measurement = conf["measurement"]
self.tags = eval(conf["tags"])
self.output_timestamp_name = conf["output_timestamp_name"]
self.influx_writer = InfluxDBClient(
url=self.url, token=self.token,
org=self.org).write_api(write_options=ASYNCHRONOUS)
def send_out(self, output_dict: Dict[str, Any],
datetime_timestamp: Any) -> None:
# Remove timestamp from output dictionary
# NOTE: timestamp MUST be in nanoseconds
timestamp = int(output_dict[self.output_timestamp_name] * 1000000000)
only_values = output_dict
del only_values[self.output_timestamp_name]
# Delete strings that cannot be written to influxdb
to_delete = []
for value in only_values:
if (isinstance(only_values[value], str)
or isinstance(only_values[value], bool)
or isinstance(only_values[value], dict)):
to_delete.append(value)
for v_d in to_delete:
del only_values[v_d]
try:
# Write to database
#print(only_values, flush=True)
self.influx_writer.write(self.bucket, self.org,
[{
"measurement": self.measurement,
"tags": self.tags,
"fields": only_values,
"time": timestamp
}])
self.influx_writer.close()
except OSError:
# If too many opened files wait one second and try again.
time.sleep(100)
print("{}: Too many opened files. Retrying now.".format(
datetime.datetime.now()),
flush=True)
self.send_out(output_dict=output_dict)
Create client that writes data in batches with 50_000 items.
"""
write_api = client.write_api(
write_options=WriteOptions(batch_size=50_000, flush_interval=10_000))
"""
Write data into InfluxDB
"""
write_api.write(bucket="my-bucket", record=data)
write_api.close()
"""
Querying max value of CBOE Volatility Index
"""
query = 'from(bucket:"my-bucket")' \
' |> range(start: 0, stop: now())' \
' |> filter(fn: (r) => r._measurement == "financial-analysis")' \
' |> max()'
result = client.query_api().query(query=query)
"""
Processing results
"""
print()
print("=== results ===")
print()
for table in result:
for record in table.records:
print('max {0:5} = {1}'.format(record.get_field(), record.get_value()))
"""
Close client
"""
client.close()
def refreshBandwidthGraphs():
startTime = datetime.now()
with open('statsByParentNode.json', 'r') as j:
parentNodes = json.loads(j.read())
with open('statsByDevice.json', 'r') as j:
devices = json.loads(j.read())
parentNodeNameDict = parentNodeNameDictPull()
print("Retrieving device statistics")
devices = getDeviceStats(devices)
print("Computing parent node statistics")
parentNodes = getParentNodeStats(parentNodes, devices)
print("Writing data to InfluxDB")
bucket = influxDBBucket
org = influxDBOrg
token = influxDBtoken
url = influxDBurl
client = InfluxDBClient(url=url, token=token, org=org)
write_api = client.write_api(write_options=SYNCHRONOUS)
queriesToSend = []
for device in devices:
bitsDownload = int(device['bitsDownloadSinceLastQuery'])
bitsUpload = int(device['bitsUploadSinceLastQuery'])
if (bitsDownload > 0) and (bitsUpload > 0):
percentUtilizationDownload = round(
(bitsDownload / round(device['downloadMax'] * 1000000)), 4)
percentUtilizationUpload = round(
(bitsUpload / round(device['uploadMax'] * 1000000)), 4)
p = Point('Bandwidth').tag("Device", device['hostname']).tag(
"ParentNode",
device['ParentNode']).tag("Type", "Device").field(
"Download", bitsDownload)
queriesToSend.append(p)
p = Point('Bandwidth').tag("Device", device['hostname']).tag(
"ParentNode",
device['ParentNode']).tag("Type", "Device").field(
"Upload", bitsUpload)
queriesToSend.append(p)
p = Point('Utilization').tag("Device", device['hostname']).tag(
"ParentNode",
device['ParentNode']).tag("Type", "Device").field(
"Download", percentUtilizationDownload)
queriesToSend.append(p)
p = Point('Utilization').tag("Device", device['hostname']).tag(
"ParentNode",
device['ParentNode']).tag("Type", "Device").field(
"Upload", percentUtilizationUpload)
queriesToSend.append(p)
for parentNode in parentNodes:
bitsDownload = int(parentNode['bitsDownloadSinceLastQuery'])
bitsUpload = int(parentNode['bitsUploadSinceLastQuery'])
if (bitsDownload > 0) and (bitsUpload > 0):
percentUtilizationDownload = round(
(bitsDownload / round(parentNode['downloadMax'] * 1000000)), 4)
percentUtilizationUpload = round(
(bitsUpload / round(parentNode['uploadMax'] * 1000000)), 4)
p = Point('Bandwidth').tag(
"Device", parentNode['parentNodeName']).tag(
"ParentNode", parentNode['parentNodeName']).tag(
"Type", "Parent Node").field("Download", bitsDownload)
queriesToSend.append(p)
p = Point('Bandwidth').tag(
"Device", parentNode['parentNodeName']).tag(
"ParentNode", parentNode['parentNodeName']).tag(
"Type", "Parent Node").field("Upload", bitsUpload)
queriesToSend.append(p)
p = Point('Utilization').tag(
"Device", parentNode['parentNodeName']).tag(
"ParentNode", parentNode['parentNodeName']).tag(
"Type",
"Parent Node").field("Download",
percentUtilizationDownload)
queriesToSend.append(p)
p = Point('Utilization').tag(
"Device", parentNode['parentNodeName']).tag(
"ParentNode", parentNode['parentNodeName']).tag(
"Type", "Parent Node").field("Upload",
percentUtilizationUpload)
queriesToSend.append(p)
write_api.write(bucket=bucket, record=queriesToSend)
print("Added " + str(len(queriesToSend)) + " points to InfluxDB.")
client.close()
with open('statsByParentNode.json', 'w') as infile:
json.dump(parentNodes, infile)
with open('statsByDevice.json', 'w') as infile:
json.dump(devices, infile)
endTime = datetime.now()
durationSeconds = round((endTime - startTime).total_seconds(), 2)
print("Graphs updated within " + str(durationSeconds) + " seconds.")
class WriteApiTestMock(BaseTest):
def setUp(self) -> None:
httpretty.enable()
httpretty.reset()
conf = influxdb_client.configuration.Configuration()
conf.host = "http://localhost"
conf.debug = False
self.influxdb_client = InfluxDBClient(url=conf.host, token="my-token")
def tearDown(self) -> None:
self.influxdb_client.close()
httpretty.disable()
def test_writes_synchronous_without_retry(self):
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=503)
self.write_client = self.influxdb_client.write_api(write_options=SYNCHRONOUS)
with self.assertRaises(ApiException) as cm:
self.write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek water_level=1 1")
exception = cm.exception
self.assertEqual("Service Unavailable", exception.reason)
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
def test_writes_asynchronous_without_retry(self):
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=503)
self.write_client = self.influxdb_client.write_api(write_options=ASYNCHRONOUS)
with self.assertRaises(ApiException) as cm:
self.write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek water_level=1 1").get()
exception = cm.exception
self.assertEqual("Service Unavailable", exception.reason)
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
def test_writes_default_tags_dict_without_tag(self):
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204)
point_settings = PointSettings(**{"id": "132-987-655", "customer": "California Miner"})
self.write_client = self.influxdb_client.write_api(write_options=SYNCHRONOUS,
point_settings=point_settings)
self.write_client.write("my-bucket", "my-org", {"measurement": "h2o", "fields": {"level": 1.0}, "time": 1})
requests = httpretty.httpretty.latest_requests
self.assertEqual(1, len(requests))
self.assertEqual("h2o,customer=California\\ Miner,id=132-987-655 level=1 1", requests[0].parsed_body)
def test_redirect(self):
from urllib3 import Retry
Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset()
Retry.DEFAULT.remove_headers_on_redirect = Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
self.influxdb_client.close()
self.influxdb_client = InfluxDBClient(url="http://localhost", token="my-token", org="my-org")
httpretty.register_uri(httpretty.POST, uri="http://localhost2/api/v2/write", status=204)
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=301,
adding_headers={'Location': 'http://localhost2/api/v2/write'})
self.write_client = self.influxdb_client.write_api(write_options=SYNCHRONOUS)
self.write_client.write("my-bucket", "my-org", {"measurement": "h2o", "fields": {"level": 1.0}, "time": 1})
requests = httpretty.httpretty.latest_requests
self.assertEqual(2, len(requests))
self.assertEqual('Token my-token', requests[0].headers['Authorization'])
self.assertEqual('Token my-token', requests[1].headers['Authorization'])
from urllib3 import Retry
Retry.DEFAULT.remove_headers_on_redirect = Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT
def test_named_tuple(self):
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204)
self.write_client = self.influxdb_client.write_api(write_options=SYNCHRONOUS)
Factory = namedtuple('Factory', ['measurement', 'position', 'customers'])
factory = Factory(measurement='factory', position="central europe", customers=123456)
self.write_client.write("my-bucket", "my-org", factory,
record_measurement_key="measurement",
record_tag_keys=["position"],
record_field_keys=["customers"])
requests = httpretty.httpretty.latest_requests
self.assertEqual(1, len(requests))
self.assertEqual("factory,position=central\\ europe customers=123456i", requests[0].parsed_body)
@pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8 or higher")
def test_data_class(self):
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204)
self.write_client = self.influxdb_client.write_api(write_options=SYNCHRONOUS)
from dataclasses import dataclass
@dataclass
class Car:
engine: str
type: str
speed: float
car = Car('12V-BT', 'sport-cars', 125.25)
self.write_client.write("my-bucket", "my-org",
record=car,
record_measurement_name="performance",
record_tag_keys=["engine", "type"],
record_field_keys=["speed"])
requests = httpretty.httpretty.latest_requests
self.assertEqual(1, len(requests))
self.assertEqual("performance,engine=12V-BT,type=sport-cars speed=125.25", requests[0].parsed_body)
class InfluxDB:
def __init__(self, config):
self._config = config
self._client = None
self._write_api = None
self._query_api = None
self._delete_api = None
self._tasks_api = None
self._organizations_api = None
self._token = None
self._org = None
self._url = None
self._bucket = None
def start(self):
"""Initialize the InfluxDB client."""
try:
influxdb_options = retrieve_options(self._config, 'influxdb2',
_INFLUXDB2_OPTIONS)
debug_options = retrieve_options(self._config, 'debug',
_DEBUG_OPTIONS)
except FailedInitialization as e:
_LOGGER.error(f"{e}")
return False
if len(influxdb_options.keys()) == 0:
raise FailedInitialization("missing 'influxdb2' options")
result = False
try:
self._bucket = influxdb_options.get('bucket', None)
self._url = influxdb_options.get('url', None)
self._token = influxdb_options.get('token', None)
self._org = influxdb_options.get('org', None)
self._client = InfluxDBClient(url=self._url,
token=self._token,
org=self._org,
enable_gzip=True)
if not self._client:
raise FailedInitialization(
f"failed to get InfluxDBClient from '{self._url}' (check url, token, and/or organization)"
)
self._write_api = self._client.write_api(write_options=SYNCHRONOUS)
self._query_api = self._client.query_api()
self._delete_api = self._client.delete_api()
self._tasks_api = self._client.tasks_api()
self._organizations_api = self._client.organizations_api()
cs_esphome_debug = os.getenv(_DEBUG_ENV_VAR,
'False').lower() in ('true', '1', 't')
try:
if cs_esphome_debug and debug_options.get(
'delete_bucket', False):
self.delete_bucket()
_LOGGER.info(
f"Deleted bucket '{self._bucket}' at '{self._url}'")
except InfluxDBBucketError as e:
raise FailedInitialization(f"{e}")
try:
if not self.connect_bucket(
cs_esphome_debug
and debug_options.get('create_bucket', False)):
raise FailedInitialization(
f"Unable to access (or create) bucket '{self._bucket}' at '{self._url}'"
)
except InfluxDBBucketError as e:
raise FailedInitialization(f"{e}")
_LOGGER.info(
f"Connected to InfluxDB: '{self._url}', bucket '{self._bucket}'"
)
result = True
except FailedInitialization as e:
_LOGGER.error(f" client {e}")
self._client = None
except NewConnectionError:
_LOGGER.error(
f"InfluxDB client unable to connect to host at {self._url}")
except ApiException as e:
_LOGGER.error(
f"InfluxDB client unable to access bucket '{self._bucket}' at {self._url}: {e.reason}"
)
except Exception as e:
_LOGGER.error(f"Unexpected exception: {e}")
finally:
return result
def stop(self):
if self._write_api:
self._write_api.close()
self._write_api = None
if self._client:
self._client.close()
self._client = None
def bucket(self):
return self._bucket
def org(self):
return self._org
def write_api(self):
return self._write_api
def query_api(self):
return self._query_api
def delete_api(self):
return self._delete_api
def tasks_api(self):
return self._tasks_api
def organizations_api(self):
return self._organizations_api
def write_point(self, measurement, tags, field, value, timestamp=None):
"""Write a single sensor to the database."""
timestamp = timestamp if timestamp is not None else int(time.time())
lp_tags = ''
separator = ''
for tag in tags:
lp_tags += f"{separator}{tag.get('t')}={tag.get('v')}"
separator = ','
lp = f"{measurement}," + lp_tags + f" {field}={value} {timestamp}"
try:
self._write_api.write(bucket=self._bucket,
record=lp,
write_precision=WritePrecision.S)
except ApiException as e:
raise InfluxDBWriteError(
f"InfluxDB client unable to write to '{self._bucket}' at {self._url}: {e.reason}"
)
except Exception as e:
raise InfluxDBWriteError(
f"Unexpected failure in write_point(): {e}")
def write_points(self, points):
"""Write a list of points to the database."""
try:
self._write_api.write(bucket=self._bucket,
record=points,
write_precision=WritePrecision.S)
except ApiException as e:
raise InfluxDBWriteError(
f"InfluxDB client unable to write to '{self._bucket}' at {self._url}: {e.reason}"
)
except Exception as e:
raise InfluxDBWriteError(
f"Unexpected failure in write_points(): {e}")
def write_batch_sensors(self, batch_sensors, timestamp=None):
"""Write a batch of sensors to the database."""
if len(batch_sensors) == 0:
return
timestamp = timestamp if timestamp is not None else int(time.time())
batch = []
for record in batch_sensors:
sensor = record.get('sensor', None)
state = record.get('state', None)
measurement = sensor.get('measurement', None)
device = sensor.get('device', None)
location = sensor.get('location', None)
precision = sensor.get('precision', None)
if measurement is None or device is None:
raise InfluxDBFormatError(
"'measurement' and/or 'device' are required")
location_tag = '' if not location or not len(
location) else f',_location={location}'
device_tag = f',_device={device}'
value = round(
state, precision) if ((precision is not None)
and isinstance(state, float)) else state
lp = f'{measurement}{device_tag}{location_tag} sample={value} {timestamp}'
batch.append(lp)
try:
self._write_api.write(bucket=self._bucket,
record=batch,
write_precision=WritePrecision.S)
except ApiException as e:
raise InfluxDBWriteError(
f"InfluxDB client unable to write to '{self._bucket}' at {self._url}: {e.reason}"
)
except Exception as e:
raise InfluxDBWriteError(
f"Unexpected failure in write_batch_sensors(): {e}")
def delete_bucket(self):
try:
buckets_api = self._client.buckets_api()
found_bucket = buckets_api.find_bucket_by_name(self._bucket)
if found_bucket:
buckets_api.delete_bucket(found_bucket)
bucket = buckets_api.find_bucket_by_name(self._bucket)
if not bucket:
return True
return False
except ApiException as e:
raise InfluxDBBucketError(
f"InfluxDB client unable to delete bucket '{self._bucket}' at {self._url}: {e.reason}"
)
except Exception as e:
raise InfluxDBBucketError(
f"Unexpected exception in delete_bucket(): {e}")
def connect_bucket(self, create_bucket=False):
try:
buckets_api = self._client.buckets_api()
bucket = buckets_api.find_bucket_by_name(self._bucket)
if bucket:
return True
if create_bucket:
bucket = buckets_api.create_bucket(bucket_name=self._bucket,
org_id=self._org,
retention_rules=None,
org=None)
if bucket:
_LOGGER.info(
f"Created bucket '{self._bucket}' at {self._url}")
return True
return False
except ApiException as e:
raise InfluxDBBucketError(
f"InfluxDB client unable to create bucket '{self._bucket}' at {self._url}: {e.reason}"
)
except Exception as e:
raise InfluxDBBucketError(
f"Unexpected exception in connect_bucket(): {e}")
class InfluxLoggingHandler(logging.Handler):
"""
InfluxLoggingHandler instances dispatch logging events to influx.
There is no need to set a Formatter.
The raw input will be passed on to the influx write api.
"""
DEFAULT_LOG_RECORD_KEYS = list(logging.makeLogRecord(
{}).__dict__.keys()) + ['message']
def __init__(self,
*,
url,
token,
org,
bucket,
client_args=None,
write_api_args=None):
"""
Initialize defaults.
The arguments `client_args` and `write_api_args` can be dicts of kwargs.
They are passed on to the InfluxDBClient and write_api calls respectively.
"""
super().__init__()
self.bucket = bucket
client_args = {} if client_args is None else client_args
self.client = InfluxDBClient(url=url,
token=token,
org=org,
**client_args)
write_api_args = {} if write_api_args is None else write_api_args
self.write_api = self.client.write_api(**write_api_args)
def __del__(self):
"""Make sure all resources are closed."""
self.close()
def close(self) -> None:
"""Close the write_api, client and logger."""
self.write_api.close()
self.client.close()
super().close()
def emit(self, record: logging.LogRecord) -> None:
"""Emit a record via the influxDB WriteApi."""
try:
message = self.format(record)
extra = self._get_extra_values(record)
return self.write_api.write(record=message, **extra)
except (KeyboardInterrupt, SystemExit):
raise
except (Exception, ):
self.handleError(record)
def _get_extra_values(self, record: logging.LogRecord) -> dict:
"""
Extract all items from the record that were injected via extra.
Example: `logging.debug(msg, extra={key: value, ...})`.
"""
extra = {'bucket': self.bucket}
extra.update({
key: value
for key, value in record.__dict__.items()
if key not in self.DEFAULT_LOG_RECORD_KEYS
})
return extra
class InfluxDBClientTestIT(BaseTest):
httpRequest = []
def tearDown(self) -> None:
super(InfluxDBClientTestIT, self).tearDown()
if hasattr(self, 'httpd'):
self.httpd.shutdown()
if hasattr(self, 'httpd_thread'):
self.httpd_thread.join()
InfluxDBClientTestIT.httpRequest = []
def test_proxy(self):
self._start_proxy_server()
self.client.close()
self.client = InfluxDBClient(
url=self.host,
token=self.auth_token,
proxy=f"http://localhost:{self.httpd.server_address[1]}",
proxy_headers={'ProxyHeader': 'Val'})
ready = self.client.ready()
self.assertEqual(ready.status, "ready")
self.assertEqual(1, len(InfluxDBClientTestIT.httpRequest))
self.assertEqual(
'Val',
InfluxDBClientTestIT.httpRequest[0].headers.get('ProxyHeader'))
def test_ping(self):
ping = self.client.ping()
self.assertTrue(ping)
def test_ping_not_running_instance(self):
client_not_running = InfluxDBClient("http://localhost:8099",
token="my-token",
debug=True)
ping = client_not_running.ping()
self.assertFalse(ping)
client_not_running.close()
def test_version(self):
version = self.client.version()
self.assertTrue(len(version) > 0)
def test_version_not_running_instance(self):
client_not_running = InfluxDBClient("http://localhost:8099",
token="my-token",
debug=True)
with self.assertRaises(NewConnectionError):
client_not_running.version()
client_not_running.close()
def test_username_password_authorization(self):
self.client.close()
self.client = InfluxDBClient(url=self.host,
username="******",
password="******",
debug=True)
self.client.query_api().query("buckets()", "my-org")
def test_query_and_debug(self):
self.client.close()
self.client = InfluxDBClient(url=self.host,
token="my-token",
debug=True)
# Query
results = self.client.query_api().query("buckets()", "my-org")
self.assertIn(
"my-bucket",
list(map(lambda record: record["name"], results[0].records)))
# Query RAW
results = self.client.query_api().query_raw("buckets()", "my-org")
self.assertIn("my-bucket", codecs.decode(results.data))
# Bucket API
results = self.client.buckets_api().find_buckets()
self.assertIn("my-bucket",
list(map(lambda bucket: bucket.name, results.buckets)))
def _start_proxy_server(self):
import http.server
import urllib.request
class ProxyHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def do_GET(self):
InfluxDBClientTestIT.httpRequest.append(self)
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.copyfile(urllib.request.urlopen(self.path), self.wfile)
self.httpd = http.server.HTTPServer(('localhost', 0),
ProxyHTTPRequestHandler)
self.httpd_thread = threading.Thread(target=self.httpd.serve_forever)
self.httpd_thread.start()
