class TestHealthMock(unittest.TestCase):
def setUp(self) -> None:
httpretty.enable()
httpretty.reset()
self.influxdb_client = InfluxDBClient(url="http://localhost",
token="my-token")
def tearDown(self) -> None:
self.influxdb_client.__del__()
httpretty.disable()
def test_without_retry(self):
httpretty.register_uri(httpretty.GET,
uri="http://localhost/health",
status=429,
adding_headers={
'Retry-After': '5',
'Content-Type': 'application/json'
},
body="{\"message\":\"Health is not working\"}")
check = self.influxdb_client.health()
self.assertTrue("Health is not working" in check.message,
msg=check.message)
self.assertEqual(check.status, "fail")
self.assertEqual(check.name, "influxdb")
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
def test_with_retry(self):
self.influxdb_client.__del__()
self.influxdb_client = InfluxDBClient(url="http://localhost",
token="my-token",
retries=Retry())
httpretty.register_uri(
httpretty.GET,
uri="http://localhost/health",
status=200,
adding_headers={'Content-Type': 'application/json'},
body=
"{\"message\":\"ready for queries and writes\", \"name\":\"influxdb\", \"status\":\"pass\"}"
)
httpretty.register_uri(httpretty.GET,
uri="http://localhost/health",
status=429,
adding_headers={
'Retry-After': '1',
'Content-Type': 'application/json'
},
body="{\"message\":\"Health is not working\"}")
health = self.influxdb_client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
self.assertEqual(2, len(httpretty.httpretty.latest_requests))
def init(self):
"""Init the connection to the InfluxDB server."""
if not self.export_enable:
return None
url = '{}://{}:{}'.format(self.protocol, self.host, self.port)
try:
client = InfluxDBClient(url=url,
enable_gzip=False,
org=self.org,
token=self.token)
except Exception as e:
logger.critical("Cannot connect to InfluxDB server '%s' (%s)" %
(url, e))
sys.exit(2)
else:
logger.info("Connected to InfluxDB server version {} ({})".format(
client.health().version,
client.health().message))
# Create the write client
write_client = client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10000,
jitter_interval=2000,
retry_interval=5000,
max_retries=5,
max_retry_delay=30000,
exponential_base=2))
return write_client
def test_health_not_running_instance(self):
client_not_running = InfluxDBClient("http://localhost:8099",
token="my-token",
debug=True)
check = client_not_running.health()
self.assertTrue("Connection refused" in check.message)
self.assertEqual(check.status, "fail")
self.assertEqual(check.name, "influxdb")
class InfluxDBClientTest(unittest.TestCase):
def tearDown(self) -> None:
if self.client:
self.client.close()
if hasattr(self, 'httpd'):
self.httpd.shutdown()
if hasattr(self, 'httpd_thread'):
self.httpd_thread.join()
def test_TrailingSlashInUrl(self):
self.client = InfluxDBClient(url="http://localhost:9999",
token="my-token",
org="my-org")
self.assertEqual('http://localhost:9999',
self.client.api_client.configuration.host)
self.client = InfluxDBClient(url="http://localhost:9999/",
token="my-token",
org="my-org")
self.assertEqual('http://localhost:9999',
self.client.api_client.configuration.host)
def test_ConnectToSelfSignedServer(self):
import http.server
import ssl
import os
# Disable unverified HTTPS requests
import urllib3
urllib3.disable_warnings()
# Configure HTTP server
self.httpd = http.server.HTTPServer(('localhost', 0),
ServerWithSelfSingedSSL)
self.httpd.socket = ssl.wrap_socket(
self.httpd.socket,
certfile=f'{os.path.dirname(__file__)}/server.pem',
server_side=True)
# Start server at background
self.httpd_thread = threading.Thread(target=self.httpd.serve_forever)
self.httpd_thread.start()
self.client = InfluxDBClient(
f"https://localhost:{self.httpd.server_address[1]}",
token="my-token",
verify_ssl=False)
health = self.client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
def connection_to_db() -> Optional[InfluxDBClient]:
"""
Создание подключения к базе данных.
Результат подключения записывается в глобальную переменную connect.
"""
# Settings
org, token, url = get_setting("database_settings")
# Connection
client = InfluxDBClient(url=url, token=token, org=org)
write_api = client.write_api(write_options=ASYNCHRONOUS)
if client.health().status != "pass":
error_log.error("Подключение к базе данных %s закончилось ошибкой %s",
url,
client.health().message)
raise DatabaseConnectionError
connect.create_connect(org=org, write_api=write_api)
event_log.info("Успешное подключение к базе данных %s", url)
return client
class Connection(object):
current = None
connections = {}
@classmethod
def tell_format(cls):
return """Connection info needed in format,
example: %flux http://localhost:9999/ --token my-token --org my-org"""
def __init__(self, url=None, token=None, org=None, debug=False):
if not url:
url = os.getenv('INFLUXDB_V2_URL')
if url is None:
raise Exception(
"Environment variable $INFLUXDB_V2_URK not set, and no url given."
)
if not token:
token = os.getenv('INFLUXDB_V2_TOKEN')
if not token:
raise Exception(
"Environment variable $INFLUXDB_V2_TOKEN not set, and no token given."
)
if not org:
org = os.getenv('INFLUXDB_V2_ORG')
if org is None:
raise Exception(
"Environment variable $INFLUXDB_V2_ORG not set, and no org given."
)
self.name = "%s@%s" % (url or "", org)
self.session = InfluxDBClient(url=url,
token=token,
org=org,
debug=debug)
self.session.health()
self.connections[repr(url)] = self
Connection.current = self
@classmethod
def set(cls, conn, token, org, displaycon, debug):
"Sets the current database connection"
if conn:
if isinstance(conn, Connection):
cls.current = conn
else:
existing = rough_dict_get(cls.connections, conn)
cls.current = existing or Connection(conn, token, org)
else:
if cls.connections:
if displaycon:
print(cls.connection_list())
else:
cls.current = Connection(conn, token, org, debug)
return cls.current
@classmethod
def connection_list(cls):
result = []
for key in sorted(cls.connections):
if cls.connections[key] == cls.current:
template = "* {}"
else:
template = " {}"
result.append(template.format(key.__repr__()))
return "\n".join(result)
def _close(cls, descriptor):
if isinstance(descriptor, Connection):
conn = descriptor
else:
conn = cls.connections.get(descriptor) or cls.connections.get(
descriptor.lower())
if not conn:
raise Exception(
"Could not close connection because it was not found amongst these: %s"
% str(cls.connections.keys()))
cls.connections.pop(conn.name)
conn.session.close()
def close(self):
self.__class__._close(self)
class InfluxDBClientTest(unittest.TestCase):
def tearDown(self) -> None:
if self.client:
self.client.close()
if hasattr(self, 'httpd'):
self.httpd.shutdown()
if hasattr(self, 'httpd_thread'):
self.httpd_thread.join()
def test_TrailingSlashInUrl(self):
self.client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org")
self.assertEqual('http://localhost:8086',
self.client.api_client.configuration.host)
self.client = InfluxDBClient(url="http://localhost:8086/",
token="my-token",
org="my-org")
self.assertEqual('http://localhost:8086',
self.client.api_client.configuration.host)
def test_ConnectToSelfSignedServer(self):
self._start_http_server()
self.client = InfluxDBClient(
f"https://localhost:{self.httpd.server_address[1]}",
token="my-token",
verify_ssl=False)
health = self.client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
def test_certificate_file(self):
self._start_http_server()
self.client = InfluxDBClient(
f"https://localhost:{self.httpd.server_address[1]}",
token="my-token",
verify_ssl=True,
ssl_ca_cert=f'{os.path.dirname(__file__)}/server.pem')
health = self.client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
def test_init_from_ini_file(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.ini')
self.assertConfig()
def test_init_from_toml_file(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.toml')
self.assertConfig()
def assertConfig(self):
self.assertEqual("http://localhost:8086", self.client.url)
self.assertEqual("my-org", self.client.org)
self.assertEqual("my-token", self.client.token)
self.assertEqual(6000, self.client.timeout)
self.assertEqual(3, len(self.client.default_tags))
self.assertEqual("132-987-655", self.client.default_tags["id"])
self.assertEqual("California Miner",
self.client.default_tags["customer"])
self.assertEqual("${env.data_center}",
self.client.default_tags["data_center"])
def test_init_from_file_ssl_default(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.ini')
self.assertTrue(self.client.api_client.configuration.verify_ssl)
def test_init_from_file_ssl(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config-disabled-ssl.ini')
self.assertFalse(self.client.api_client.configuration.verify_ssl)
def test_init_from_env_ssl_default(self):
if os.getenv("INFLUXDB_V2_VERIFY_SSL"):
del os.environ["INFLUXDB_V2_VERIFY_SSL"]
self.client = InfluxDBClient.from_env_properties()
self.assertTrue(self.client.api_client.configuration.verify_ssl)
def test_init_from_env_ssl(self):
os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path"
self.client = InfluxDBClient.from_env_properties()
self.assertEqual("/my/custom/path",
self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_file_ssl_ca_cert_default(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.ini')
self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_file_ssl_ca_cert(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config-ssl-ca-cert.ini')
self.assertEqual("/path/to/my/cert",
self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_env_ssl_ca_cert_default(self):
if os.getenv("INFLUXDB_V2_SSL_CA_CERT"):
del os.environ["INFLUXDB_V2_SSL_CA_CERT"]
self.client = InfluxDBClient.from_env_properties()
self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_env_ssl_ca_cert(self):
os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path/to/cert"
self.client = InfluxDBClient.from_env_properties()
self.assertEqual("/my/custom/path/to/cert",
self.client.api_client.configuration.ssl_ca_cert)
def _start_http_server(self):
import http.server
import ssl
# Disable unverified HTTPS requests
import urllib3
urllib3.disable_warnings()
# Configure HTTP server
self.httpd = http.server.HTTPServer(('localhost', 0),
ServerWithSelfSingedSSL)
self.httpd.socket = ssl.wrap_socket(
self.httpd.socket,
certfile=f'{os.path.dirname(__file__)}/server.pem',
server_side=True)
# Start server at background
self.httpd_thread = threading.Thread(target=self.httpd.serve_forever)
self.httpd_thread.start()
class InfluxDBClientTest(unittest.TestCase):
def tearDown(self) -> None:
if self.client:
self.client.close()
if hasattr(self, 'httpd'):
self.httpd.shutdown()
if hasattr(self, 'httpd_thread'):
self.httpd_thread.join()
def test_default_conf(self):
self.client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org")
self.assertIsNotNone(
self.client.api_client.configuration.connection_pool_maxsize)
def test_TrailingSlashInUrl(self):
self.client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org")
self.assertEqual('http://localhost:8086',
self.client.api_client.configuration.host)
self.client = InfluxDBClient(url="http://localhost:8086/",
token="my-token",
org="my-org")
self.assertEqual('http://localhost:8086',
self.client.api_client.configuration.host)
def test_ConnectToSelfSignedServer(self):
self._start_http_server()
self.client = InfluxDBClient(
f"https://localhost:{self.httpd.server_address[1]}",
token="my-token",
verify_ssl=False)
health = self.client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
def test_certificate_file(self):
self._start_http_server()
self.client = InfluxDBClient(
f"https://localhost:{self.httpd.server_address[1]}",
token="my-token",
verify_ssl=True,
ssl_ca_cert=f'{os.path.dirname(__file__)}/server.pem')
health = self.client.health()
self.assertEqual(health.message, 'ready for queries and writes')
self.assertEqual(health.status, "pass")
self.assertEqual(health.name, "influxdb")
def test_init_from_ini_file(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.ini')
self.assertConfig()
def test_init_from_toml_file(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.toml')
self.assertConfig()
def assertConfig(self):
self.assertEqual("http://localhost:8086", self.client.url)
self.assertEqual("my-org", self.client.org)
self.assertEqual("my-token", self.client.token)
self.assertEqual(6000, self.client.api_client.configuration.timeout)
self.assertEqual(3, len(self.client.default_tags))
self.assertEqual("132-987-655", self.client.default_tags["id"])
self.assertEqual("California Miner",
self.client.default_tags["customer"])
self.assertEqual("${env.data_center}",
self.client.default_tags["data_center"])
self.assertEqual(
55, self.client.api_client.configuration.connection_pool_maxsize)
def test_init_from_file_ssl_default(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.ini')
self.assertTrue(self.client.api_client.configuration.verify_ssl)
def test_init_from_file_ssl(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config-disabled-ssl.ini')
self.assertFalse(self.client.api_client.configuration.verify_ssl)
def test_init_from_env_ssl_default(self):
if os.getenv("INFLUXDB_V2_VERIFY_SSL"):
del os.environ["INFLUXDB_V2_VERIFY_SSL"]
self.client = InfluxDBClient.from_env_properties()
self.assertTrue(self.client.api_client.configuration.verify_ssl)
def test_init_from_env_ssl(self):
os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path"
self.client = InfluxDBClient.from_env_properties()
self.assertEqual("/my/custom/path",
self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_file_ssl_ca_cert_default(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config.ini')
self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_file_ssl_ca_cert(self):
self.client = InfluxDBClient.from_config_file(
f'{os.path.dirname(__file__)}/config-ssl-ca-cert.ini')
self.assertEqual("/path/to/my/cert",
self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_env_ssl_ca_cert_default(self):
if os.getenv("INFLUXDB_V2_SSL_CA_CERT"):
del os.environ["INFLUXDB_V2_SSL_CA_CERT"]
self.client = InfluxDBClient.from_env_properties()
self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_env_ssl_ca_cert(self):
os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path/to/cert"
self.client = InfluxDBClient.from_env_properties()
self.assertEqual("/my/custom/path/to/cert",
self.client.api_client.configuration.ssl_ca_cert)
def test_init_from_env_connection_pool_maxsize(self):
os.environ["INFLUXDB_V2_CONNECTION_POOL_MAXSIZE"] = "29"
self.client = InfluxDBClient.from_env_properties()
self.assertEqual(
29, self.client.api_client.configuration.connection_pool_maxsize)
def _start_http_server(self):
import http.server
import ssl
# Disable unverified HTTPS requests
import urllib3
urllib3.disable_warnings()
# Configure HTTP server
self.httpd = http.server.HTTPServer(('localhost', 0),
ServerWithSelfSingedSSL)
self.httpd.socket = ssl.wrap_socket(
self.httpd.socket,
certfile=f'{os.path.dirname(__file__)}/server.pem',
server_side=True)
# Start server at background
self.httpd_thread = threading.Thread(target=self.httpd.serve_forever)
self.httpd_thread.start()
def test_write_context_manager(self):
with InfluxDBClient.from_env_properties(self.debug) as self.client:
api_client = self.client.api_client
with self.client.write_api(write_options=WriteOptions(
write_type=WriteType.batching)) as write_api:
write_api_test = write_api
write_api.write(bucket="my-bucket",
record=Point("h2o_feet").tag(
"location", "coyote_creek").field(
"level water_level", 5.0))
self.assertIsNotNone(write_api._subject)
self.assertIsNotNone(write_api._disposable)
self.assertIsNone(write_api_test._subject)
self.assertIsNone(write_api_test._disposable)
self.assertIsNotNone(self.client.api_client)
self.assertIsNotNone(
self.client.api_client.rest_client.pool_manager)
self.assertIsNone(api_client._pool)
self.assertIsNone(self.client.api_client)
def test_injector_provides_healthy_influxdb_client(
influx_v2_client: InfluxDBClient):
assert influx_v2_client.health().status == 'pass'
class TimeseriesClient:
host: Union[str, None]
port: int
token: Union[str, None]
organization: str
def __init__(
self,
host: str = None,
port: int = None,
organization: str = "GEWV",
token: str = None,
client: InfluxDBClient = None,
verify_ssl: bool = True,
):
if client is None:
if host is None:
raise Exception(
"Missing Host Address for Timeseries DB Client.")
if port is None:
raise Exception("Missing Port for Timeseries DB Client.")
if token is None:
raise Exception("Missing Token for Timeseries DB Client.")
protocol = "https" if verify_ssl else "http"
self._client = InfluxDBClient(
url=f"{protocol}://{host}:{port}",
token=token,
verify_ssl=verify_ssl,
)
if len(organization) != 16:
# receive id of the org and store the info
self._org_api = self._client.organizations_api()
self._org_id = self.get_org_id_by_name(org_name=organization)
if self._org_id is None:
raise Exception(
f"The organization {organization} dont exists in InfluxDB. Break execution."
)
self._client.org = self._org_id
else:
self._client.org = organization
else:
self._client = client
self._org_api = self._client.organizations_api()
self._write_api = self._client.write_api(write_options=SYNCHRONOUS)
self._query_api = self._client.query_api()
self._bucket_api = self._client.buckets_api()
self._grafana_api = GrafanaApi(host=host, port=3000, use_tls=False)
@staticmethod
def from_env_properties():
client = InfluxDBClient.from_env_properties()
return TimeseriesClient(client=client)
def health(self):
return self._client.health()
def get_org_id_by_name(self, org_name: str) -> Union[str, None]:
orgs: List[Organization] = self._org_api.find_organizations()
for org in orgs:
if org.name == org_name:
return org.id
return None
def create_bucket(self, bucket: str):
try:
self._bucket_api.create_bucket(bucket_name=bucket)
except ApiException as err:
if err.status != 422:
raise
def exist_bucket(self, bucket: str):
return self._bucket_api.find_bucket_by_name(bucket_name=bucket)
def get_bucket_by_name(self, bucket_name: str):
return self._bucket_api.find_bucket_by_name(bucket_name=bucket_name)
def delete_bucket(self, bucket: str):
bucket_id = self.get_bucket_by_name(bucket_name=bucket)
return self._bucket_api.delete_bucket(bucket=bucket_id)
def get_grafana_orgs(self) -> List[GrafanaOrganization]:
return self._grafana_api.get_organizations()
def get_grafana_org(self, org_name: str) -> GrafanaOrganization:
return self._grafana_api.get_organization_by_name(org_name=org_name)
def create_grafana_org(self, org_name: str):
return self._grafana_api.create_organization(org_name=org_name)
def delete_grafana_org(self, org_name: str):
org = self.get_grafana_org(org_name=org_name)
if org is None:
raise Exception(
f"Cant delete grafana org {org_name}. Org not exist!")
return self._grafana_api.delete_organization(org["id"])
def create_project(self, project_name: str):
# Steps
# 1. create new bucket
# 2. create token for bucket
# 3. create new org in grafana
# 4. create new source in grafana
pass
def get_points(
self,
**kwargs,
) -> List[FluxTable]:
if not self.health:
raise Exception("Influx DB is not reachable or unhealthy.")
tables = self._query_api.query(query=self.build_query(**kwargs))
return tables
def get_dataframe(self, **kwargs):
return self.query_dataframe(flux_query=self.build_query(**kwargs))
def query_dataframe(
self,
flux_query: str,
):
"""
with this function you can send a own query to InfluxDB and
you will get back a dataframe with datetimeindex
"""
if not self.health:
raise Exception("Influx DB is not reachable or unhealthy.")
df = cast(
DataFrame,
self._query_api.query_data_frame(query=flux_query),
)
if "_time" in df.columns:
df = df.set_index(pd.to_datetime(df["_time"]))
return df
def write_points(self, project: str, points: List[Point]):
self._write_api.write(bucket=project, record=points)
def write_a_dataframe(
self,
project: str,
measurement_name: str,
dataframe: pd.DataFrame,
tag_columns: List[str] = [],
additional_tags: Dict[str, str] = None,
):
"""
Write a pandas dataframe to the influx db. You can define some
tags, that are appended to every entry.
"""
if additional_tags is None:
self._write_api.write(
bucket=project,
record=dataframe,
data_frame_measurement_name=measurement_name,
data_frame_tag_columns=tag_columns,
)
return
tags_dataframe = pd.DataFrame(index=dataframe.index)
# create the dataframe with the tags
for tag_name, tag in additional_tags.items():
tag_columns.append(tag_name)
tags_dataframe[tag_name] = [tag] * len(dataframe)
combined_frames = pd.concat([dataframe, tags_dataframe], axis=1)
self._write_api.write(
bucket=project,
record=combined_frames,
data_frame_measurement_name=measurement_name,
data_frame_tag_columns=tag_columns,
)
def build_query(
self,
project: str,
fields: Dict[str, str] = {},
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
precision: str = "5m",
) -> str:
query = f"""
from(bucket: "{project}")
"""
if start_time is not None and end_time is not None:
self.test_datetime(start_time)
self.test_datetime(end_time)
query += f"""
|> range(start: {start_time.isoformat()}, stop: {end_time.isoformat()})
"""
elif start_time is not None:
self.test_datetime(start_time)
query += f"""
|> range(start: {start_time.isoformat()})
"""
elif end_time is not None:
self.test_datetime(end_time)
query += f"""
|> range(stop: {end_time.isoformat()})
"""
for f, v in fields.items():
query += f"""
|> filter(fn: (r) => r["{f}"] == "{v}")
"""
query += f"""
|> aggregateWindow(every: {precision}, fn: mean, createEmpty: true)
|> yield(name: "mean")
"""
return query
@staticmethod
def test_datetime(dt: datetime):
if not isinstance(dt, datetime):
raise Exception(
f"The delivered datetime {dt} is not from type datetime.")
if dt.tzinfo is None:
raise Exception(
f"The time {dt.isoformat()} has no timezone info. That is necassary."
)
