class TestHealthMock(unittest.TestCase): def setUp(self) -> None: httpretty.enable() httpretty.reset() self.influxdb_client = InfluxDBClient(url="http://localhost", token="my-token") def tearDown(self) -> None: self.influxdb_client.__del__() httpretty.disable() def test_without_retry(self): httpretty.register_uri(httpretty.GET, uri="http://localhost/health", status=429, adding_headers={ 'Retry-After': '5', 'Content-Type': 'application/json' }, body="{\"message\":\"Health is not working\"}") check = self.influxdb_client.health() self.assertTrue("Health is not working" in check.message, msg=check.message) self.assertEqual(check.status, "fail") self.assertEqual(check.name, "influxdb") self.assertEqual(1, len(httpretty.httpretty.latest_requests)) def test_with_retry(self): self.influxdb_client.__del__() self.influxdb_client = InfluxDBClient(url="http://localhost", token="my-token", retries=Retry()) httpretty.register_uri( httpretty.GET, uri="http://localhost/health", status=200, adding_headers={'Content-Type': 'application/json'}, body= "{\"message\":\"ready for queries and writes\", \"name\":\"influxdb\", \"status\":\"pass\"}" ) httpretty.register_uri(httpretty.GET, uri="http://localhost/health", status=429, adding_headers={ 'Retry-After': '1', 'Content-Type': 'application/json' }, body="{\"message\":\"Health is not working\"}") health = self.influxdb_client.health() self.assertEqual(health.message, 'ready for queries and writes') self.assertEqual(health.status, "pass") self.assertEqual(health.name, "influxdb") self.assertEqual(2, len(httpretty.httpretty.latest_requests))
def init(self): """Init the connection to the InfluxDB server.""" if not self.export_enable: return None url = '{}://{}:{}'.format(self.protocol, self.host, self.port) try: client = InfluxDBClient(url=url, enable_gzip=False, org=self.org, token=self.token) except Exception as e: logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e)) sys.exit(2) else: logger.info("Connected to InfluxDB server version {} ({})".format( client.health().version, client.health().message)) # Create the write client write_client = client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10000, jitter_interval=2000, retry_interval=5000, max_retries=5, max_retry_delay=30000, exponential_base=2)) return write_client
def test_health_not_running_instance(self): client_not_running = InfluxDBClient("http://localhost:8099", token="my-token", debug=True) check = client_not_running.health() self.assertTrue("Connection refused" in check.message) self.assertEqual(check.status, "fail") self.assertEqual(check.name, "influxdb")
class InfluxDBClientTest(unittest.TestCase): def tearDown(self) -> None: if self.client: self.client.close() if hasattr(self, 'httpd'): self.httpd.shutdown() if hasattr(self, 'httpd_thread'): self.httpd_thread.join() def test_TrailingSlashInUrl(self): self.client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org") self.assertEqual('http://localhost:9999', self.client.api_client.configuration.host) self.client = InfluxDBClient(url="http://localhost:9999/", token="my-token", org="my-org") self.assertEqual('http://localhost:9999', self.client.api_client.configuration.host) def test_ConnectToSelfSignedServer(self): import http.server import ssl import os # Disable unverified HTTPS requests import urllib3 urllib3.disable_warnings() # Configure HTTP server self.httpd = http.server.HTTPServer(('localhost', 0), ServerWithSelfSingedSSL) self.httpd.socket = ssl.wrap_socket( self.httpd.socket, certfile=f'{os.path.dirname(__file__)}/server.pem', server_side=True) # Start server at background self.httpd_thread = threading.Thread(target=self.httpd.serve_forever) self.httpd_thread.start() self.client = InfluxDBClient( f"https://localhost:{self.httpd.server_address[1]}", token="my-token", verify_ssl=False) health = self.client.health() self.assertEqual(health.message, 'ready for queries and writes') self.assertEqual(health.status, "pass") self.assertEqual(health.name, "influxdb")
def connection_to_db() -> Optional[InfluxDBClient]: """ Создание подключения к базе данных. Результат подключения записывается в глобальную переменную connect. """ # Settings org, token, url = get_setting("database_settings") # Connection client = InfluxDBClient(url=url, token=token, org=org) write_api = client.write_api(write_options=ASYNCHRONOUS) if client.health().status != "pass": error_log.error("Подключение к базе данных %s закончилось ошибкой %s", url, client.health().message) raise DatabaseConnectionError connect.create_connect(org=org, write_api=write_api) event_log.info("Успешное подключение к базе данных %s", url) return client
class Connection(object): current = None connections = {} @classmethod def tell_format(cls): return """Connection info needed in format, example: %flux http://localhost:9999/ --token my-token --org my-org""" def __init__(self, url=None, token=None, org=None, debug=False): if not url: url = os.getenv('INFLUXDB_V2_URL') if url is None: raise Exception( "Environment variable $INFLUXDB_V2_URK not set, and no url given." ) if not token: token = os.getenv('INFLUXDB_V2_TOKEN') if not token: raise Exception( "Environment variable $INFLUXDB_V2_TOKEN not set, and no token given." ) if not org: org = os.getenv('INFLUXDB_V2_ORG') if org is None: raise Exception( "Environment variable $INFLUXDB_V2_ORG not set, and no org given." ) self.name = "%s@%s" % (url or "", org) self.session = InfluxDBClient(url=url, token=token, org=org, debug=debug) self.session.health() self.connections[repr(url)] = self Connection.current = self @classmethod def set(cls, conn, token, org, displaycon, debug): "Sets the current database connection" if conn: if isinstance(conn, Connection): cls.current = conn else: existing = rough_dict_get(cls.connections, conn) cls.current = existing or Connection(conn, token, org) else: if cls.connections: if displaycon: print(cls.connection_list()) else: cls.current = Connection(conn, token, org, debug) return cls.current @classmethod def connection_list(cls): result = [] for key in sorted(cls.connections): if cls.connections[key] == cls.current: template = "* {}" else: template = " {}" result.append(template.format(key.__repr__())) return "\n".join(result) def _close(cls, descriptor): if isinstance(descriptor, Connection): conn = descriptor else: conn = cls.connections.get(descriptor) or cls.connections.get( descriptor.lower()) if not conn: raise Exception( "Could not close connection because it was not found amongst these: %s" % str(cls.connections.keys())) cls.connections.pop(conn.name) conn.session.close() def close(self): self.__class__._close(self)
class InfluxDBClientTest(unittest.TestCase): def tearDown(self) -> None: if self.client: self.client.close() if hasattr(self, 'httpd'): self.httpd.shutdown() if hasattr(self, 'httpd_thread'): self.httpd_thread.join() def test_TrailingSlashInUrl(self): self.client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") self.assertEqual('http://localhost:8086', self.client.api_client.configuration.host) self.client = InfluxDBClient(url="http://localhost:8086/", token="my-token", org="my-org") self.assertEqual('http://localhost:8086', self.client.api_client.configuration.host) def test_ConnectToSelfSignedServer(self): self._start_http_server() self.client = InfluxDBClient( f"https://localhost:{self.httpd.server_address[1]}", token="my-token", verify_ssl=False) health = self.client.health() self.assertEqual(health.message, 'ready for queries and writes') self.assertEqual(health.status, "pass") self.assertEqual(health.name, "influxdb") def test_certificate_file(self): self._start_http_server() self.client = InfluxDBClient( f"https://localhost:{self.httpd.server_address[1]}", token="my-token", verify_ssl=True, ssl_ca_cert=f'{os.path.dirname(__file__)}/server.pem') health = self.client.health() self.assertEqual(health.message, 'ready for queries and writes') self.assertEqual(health.status, "pass") self.assertEqual(health.name, "influxdb") def test_init_from_ini_file(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.ini') self.assertConfig() def test_init_from_toml_file(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.toml') self.assertConfig() def assertConfig(self): self.assertEqual("http://localhost:8086", self.client.url) self.assertEqual("my-org", self.client.org) self.assertEqual("my-token", self.client.token) self.assertEqual(6000, self.client.timeout) self.assertEqual(3, len(self.client.default_tags)) self.assertEqual("132-987-655", self.client.default_tags["id"]) self.assertEqual("California Miner", self.client.default_tags["customer"]) self.assertEqual("${env.data_center}", self.client.default_tags["data_center"]) def test_init_from_file_ssl_default(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.ini') self.assertTrue(self.client.api_client.configuration.verify_ssl) def test_init_from_file_ssl(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config-disabled-ssl.ini') self.assertFalse(self.client.api_client.configuration.verify_ssl) def test_init_from_env_ssl_default(self): if os.getenv("INFLUXDB_V2_VERIFY_SSL"): del os.environ["INFLUXDB_V2_VERIFY_SSL"] self.client = InfluxDBClient.from_env_properties() self.assertTrue(self.client.api_client.configuration.verify_ssl) def test_init_from_env_ssl(self): os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path" self.client = InfluxDBClient.from_env_properties() self.assertEqual("/my/custom/path", self.client.api_client.configuration.ssl_ca_cert) def test_init_from_file_ssl_ca_cert_default(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.ini') self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert) def test_init_from_file_ssl_ca_cert(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config-ssl-ca-cert.ini') self.assertEqual("/path/to/my/cert", self.client.api_client.configuration.ssl_ca_cert) def test_init_from_env_ssl_ca_cert_default(self): if os.getenv("INFLUXDB_V2_SSL_CA_CERT"): del os.environ["INFLUXDB_V2_SSL_CA_CERT"] self.client = InfluxDBClient.from_env_properties() self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert) def test_init_from_env_ssl_ca_cert(self): os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path/to/cert" self.client = InfluxDBClient.from_env_properties() self.assertEqual("/my/custom/path/to/cert", self.client.api_client.configuration.ssl_ca_cert) def _start_http_server(self): import http.server import ssl # Disable unverified HTTPS requests import urllib3 urllib3.disable_warnings() # Configure HTTP server self.httpd = http.server.HTTPServer(('localhost', 0), ServerWithSelfSingedSSL) self.httpd.socket = ssl.wrap_socket( self.httpd.socket, certfile=f'{os.path.dirname(__file__)}/server.pem', server_side=True) # Start server at background self.httpd_thread = threading.Thread(target=self.httpd.serve_forever) self.httpd_thread.start()
class InfluxDBClientTest(unittest.TestCase): def tearDown(self) -> None: if self.client: self.client.close() if hasattr(self, 'httpd'): self.httpd.shutdown() if hasattr(self, 'httpd_thread'): self.httpd_thread.join() def test_default_conf(self): self.client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") self.assertIsNotNone( self.client.api_client.configuration.connection_pool_maxsize) def test_TrailingSlashInUrl(self): self.client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") self.assertEqual('http://localhost:8086', self.client.api_client.configuration.host) self.client = InfluxDBClient(url="http://localhost:8086/", token="my-token", org="my-org") self.assertEqual('http://localhost:8086', self.client.api_client.configuration.host) def test_ConnectToSelfSignedServer(self): self._start_http_server() self.client = InfluxDBClient( f"https://localhost:{self.httpd.server_address[1]}", token="my-token", verify_ssl=False) health = self.client.health() self.assertEqual(health.message, 'ready for queries and writes') self.assertEqual(health.status, "pass") self.assertEqual(health.name, "influxdb") def test_certificate_file(self): self._start_http_server() self.client = InfluxDBClient( f"https://localhost:{self.httpd.server_address[1]}", token="my-token", verify_ssl=True, ssl_ca_cert=f'{os.path.dirname(__file__)}/server.pem') health = self.client.health() self.assertEqual(health.message, 'ready for queries and writes') self.assertEqual(health.status, "pass") self.assertEqual(health.name, "influxdb") def test_init_from_ini_file(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.ini') self.assertConfig() def test_init_from_toml_file(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.toml') self.assertConfig() def assertConfig(self): self.assertEqual("http://localhost:8086", self.client.url) self.assertEqual("my-org", self.client.org) self.assertEqual("my-token", self.client.token) self.assertEqual(6000, self.client.api_client.configuration.timeout) self.assertEqual(3, len(self.client.default_tags)) self.assertEqual("132-987-655", self.client.default_tags["id"]) self.assertEqual("California Miner", self.client.default_tags["customer"]) self.assertEqual("${env.data_center}", self.client.default_tags["data_center"]) self.assertEqual( 55, self.client.api_client.configuration.connection_pool_maxsize) def test_init_from_file_ssl_default(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.ini') self.assertTrue(self.client.api_client.configuration.verify_ssl) def test_init_from_file_ssl(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config-disabled-ssl.ini') self.assertFalse(self.client.api_client.configuration.verify_ssl) def test_init_from_env_ssl_default(self): if os.getenv("INFLUXDB_V2_VERIFY_SSL"): del os.environ["INFLUXDB_V2_VERIFY_SSL"] self.client = InfluxDBClient.from_env_properties() self.assertTrue(self.client.api_client.configuration.verify_ssl) def test_init_from_env_ssl(self): os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path" self.client = InfluxDBClient.from_env_properties() self.assertEqual("/my/custom/path", self.client.api_client.configuration.ssl_ca_cert) def test_init_from_file_ssl_ca_cert_default(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config.ini') self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert) def test_init_from_file_ssl_ca_cert(self): self.client = InfluxDBClient.from_config_file( f'{os.path.dirname(__file__)}/config-ssl-ca-cert.ini') self.assertEqual("/path/to/my/cert", self.client.api_client.configuration.ssl_ca_cert) def test_init_from_env_ssl_ca_cert_default(self): if os.getenv("INFLUXDB_V2_SSL_CA_CERT"): del os.environ["INFLUXDB_V2_SSL_CA_CERT"] self.client = InfluxDBClient.from_env_properties() self.assertIsNone(self.client.api_client.configuration.ssl_ca_cert) def test_init_from_env_ssl_ca_cert(self): os.environ["INFLUXDB_V2_SSL_CA_CERT"] = "/my/custom/path/to/cert" self.client = InfluxDBClient.from_env_properties() self.assertEqual("/my/custom/path/to/cert", self.client.api_client.configuration.ssl_ca_cert) def test_init_from_env_connection_pool_maxsize(self): os.environ["INFLUXDB_V2_CONNECTION_POOL_MAXSIZE"] = "29" self.client = InfluxDBClient.from_env_properties() self.assertEqual( 29, self.client.api_client.configuration.connection_pool_maxsize) def _start_http_server(self): import http.server import ssl # Disable unverified HTTPS requests import urllib3 urllib3.disable_warnings() # Configure HTTP server self.httpd = http.server.HTTPServer(('localhost', 0), ServerWithSelfSingedSSL) self.httpd.socket = ssl.wrap_socket( self.httpd.socket, certfile=f'{os.path.dirname(__file__)}/server.pem', server_side=True) # Start server at background self.httpd_thread = threading.Thread(target=self.httpd.serve_forever) self.httpd_thread.start() def test_write_context_manager(self): with InfluxDBClient.from_env_properties(self.debug) as self.client: api_client = self.client.api_client with self.client.write_api(write_options=WriteOptions( write_type=WriteType.batching)) as write_api: write_api_test = write_api write_api.write(bucket="my-bucket", record=Point("h2o_feet").tag( "location", "coyote_creek").field( "level water_level", 5.0)) self.assertIsNotNone(write_api._subject) self.assertIsNotNone(write_api._disposable) self.assertIsNone(write_api_test._subject) self.assertIsNone(write_api_test._disposable) self.assertIsNotNone(self.client.api_client) self.assertIsNotNone( self.client.api_client.rest_client.pool_manager) self.assertIsNone(api_client._pool) self.assertIsNone(self.client.api_client)
def test_injector_provides_healthy_influxdb_client( influx_v2_client: InfluxDBClient): assert influx_v2_client.health().status == 'pass'
class TimeseriesClient: host: Union[str, None] port: int token: Union[str, None] organization: str def __init__( self, host: str = None, port: int = None, organization: str = "GEWV", token: str = None, client: InfluxDBClient = None, verify_ssl: bool = True, ): if client is None: if host is None: raise Exception( "Missing Host Address for Timeseries DB Client.") if port is None: raise Exception("Missing Port for Timeseries DB Client.") if token is None: raise Exception("Missing Token for Timeseries DB Client.") protocol = "https" if verify_ssl else "http" self._client = InfluxDBClient( url=f"{protocol}://{host}:{port}", token=token, verify_ssl=verify_ssl, ) if len(organization) != 16: # receive id of the org and store the info self._org_api = self._client.organizations_api() self._org_id = self.get_org_id_by_name(org_name=organization) if self._org_id is None: raise Exception( f"The organization {organization} dont exists in InfluxDB. Break execution." ) self._client.org = self._org_id else: self._client.org = organization else: self._client = client self._org_api = self._client.organizations_api() self._write_api = self._client.write_api(write_options=SYNCHRONOUS) self._query_api = self._client.query_api() self._bucket_api = self._client.buckets_api() self._grafana_api = GrafanaApi(host=host, port=3000, use_tls=False) @staticmethod def from_env_properties(): client = InfluxDBClient.from_env_properties() return TimeseriesClient(client=client) def health(self): return self._client.health() def get_org_id_by_name(self, org_name: str) -> Union[str, None]: orgs: List[Organization] = self._org_api.find_organizations() for org in orgs: if org.name == org_name: return org.id return None def create_bucket(self, bucket: str): try: self._bucket_api.create_bucket(bucket_name=bucket) except ApiException as err: if err.status != 422: raise def exist_bucket(self, bucket: str): return self._bucket_api.find_bucket_by_name(bucket_name=bucket) def get_bucket_by_name(self, bucket_name: str): return self._bucket_api.find_bucket_by_name(bucket_name=bucket_name) def delete_bucket(self, bucket: str): bucket_id = self.get_bucket_by_name(bucket_name=bucket) return self._bucket_api.delete_bucket(bucket=bucket_id) def get_grafana_orgs(self) -> List[GrafanaOrganization]: return self._grafana_api.get_organizations() def get_grafana_org(self, org_name: str) -> GrafanaOrganization: return self._grafana_api.get_organization_by_name(org_name=org_name) def create_grafana_org(self, org_name: str): return self._grafana_api.create_organization(org_name=org_name) def delete_grafana_org(self, org_name: str): org = self.get_grafana_org(org_name=org_name) if org is None: raise Exception( f"Cant delete grafana org {org_name}. Org not exist!") return self._grafana_api.delete_organization(org["id"]) def create_project(self, project_name: str): # Steps # 1. create new bucket # 2. create token for bucket # 3. create new org in grafana # 4. create new source in grafana pass def get_points( self, **kwargs, ) -> List[FluxTable]: if not self.health: raise Exception("Influx DB is not reachable or unhealthy.") tables = self._query_api.query(query=self.build_query(**kwargs)) return tables def get_dataframe(self, **kwargs): return self.query_dataframe(flux_query=self.build_query(**kwargs)) def query_dataframe( self, flux_query: str, ): """ with this function you can send a own query to InfluxDB and you will get back a dataframe with datetimeindex """ if not self.health: raise Exception("Influx DB is not reachable or unhealthy.") df = cast( DataFrame, self._query_api.query_data_frame(query=flux_query), ) if "_time" in df.columns: df = df.set_index(pd.to_datetime(df["_time"])) return df def write_points(self, project: str, points: List[Point]): self._write_api.write(bucket=project, record=points) def write_a_dataframe( self, project: str, measurement_name: str, dataframe: pd.DataFrame, tag_columns: List[str] = [], additional_tags: Dict[str, str] = None, ): """ Write a pandas dataframe to the influx db. You can define some tags, that are appended to every entry. """ if additional_tags is None: self._write_api.write( bucket=project, record=dataframe, data_frame_measurement_name=measurement_name, data_frame_tag_columns=tag_columns, ) return tags_dataframe = pd.DataFrame(index=dataframe.index) # create the dataframe with the tags for tag_name, tag in additional_tags.items(): tag_columns.append(tag_name) tags_dataframe[tag_name] = [tag] * len(dataframe) combined_frames = pd.concat([dataframe, tags_dataframe], axis=1) self._write_api.write( bucket=project, record=combined_frames, data_frame_measurement_name=measurement_name, data_frame_tag_columns=tag_columns, ) def build_query( self, project: str, fields: Dict[str, str] = {}, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None, precision: str = "5m", ) -> str: query = f""" from(bucket: "{project}") """ if start_time is not None and end_time is not None: self.test_datetime(start_time) self.test_datetime(end_time) query += f""" |> range(start: {start_time.isoformat()}, stop: {end_time.isoformat()}) """ elif start_time is not None: self.test_datetime(start_time) query += f""" |> range(start: {start_time.isoformat()}) """ elif end_time is not None: self.test_datetime(end_time) query += f""" |> range(stop: {end_time.isoformat()}) """ for f, v in fields.items(): query += f""" |> filter(fn: (r) => r["{f}"] == "{v}") """ query += f""" |> aggregateWindow(every: {precision}, fn: mean, createEmpty: true) |> yield(name: "mean") """ return query @staticmethod def test_datetime(dt: datetime): if not isinstance(dt, datetime): raise Exception( f"The delivered datetime {dt} is not from type datetime.") if dt.tzinfo is None: raise Exception( f"The time {dt.isoformat()} has no timezone info. That is necassary." )