def __init__(self, url="http://localhost:8086", settingsPath="./emon_config.yml",batchProcess=True): self.bucket = "sensors" self.client = InfluxDBClient(url, token="my-token", org="my-org") if(batchProcess): self.write_api = self.client.write_api(write_options=WriteOptions(batch_size=50_000, flush_interval=10_000)) else: self.write_api = self.client.write_api(write_options=WriteOptions(batch_size=50, flush_interval=10)) self.lineNumber = -1 settingsFile = open(settingsPath, 'r') self.settings = yaml.full_load(settingsFile) self.dispatch = { 'rain' : self.rainMessage, 'temp' : self.temperatureMessage, 'disp' : self.dispMessage, 'pulse': self.pulseMessage, 'hws' : self.HWSMessage, 'wtr' : self.waterMessage, 'scl' : self.scaleMessage, 'bat' : self.batteryMessage, 'inv' : self.inverterMessage, 'bee' : self.beeMessage, 'air' : self.airMessage, 'leaf' : self.leafMessage, 'other': self.otherMessage }
def write_api( self, write_options=WriteOptions(), point_settings=PointSettings() ) -> WriteApi: """ Create a Write API instance. :param point_settings: :param write_options: write api configuration :return: write api instance """ #WriteApi 객체를 생성해서 리턴하는것임. 인자로 받는 write_options와 point_settings 모두 클래스네. 원형을 하나만 찾아보자! """ class WriteOptions(object): def __init__(self, write_type: WriteType = WriteType.batching, batch_size=1_000, flush_interval=1_000, jitter_interval=0, retry_interval=5_000, max_retries=3, max_retry_delay=180_000, exponential_base=5, write_scheduler=ThreadPoolScheduler(max_workers=1)) -> None: self.write_type = write_type self.batch_size = batch_size self.flush_interval = flush_interval self.jitter_interval = jitter_interval self.retry_interval = retry_interval self.max_retries = max_retries self.max_retry_delay = max_retry_delay self.exponential_base = exponential_base self.write_scheduler = write_scheduler """ # 보면 위 객체는 그냥 값들만 담고있음. 그니까 어떻게보면 javascript식 객체생성을 하는 느낌이네. return WriteApi(influxdb_client=self, write_options=write_options, point_settings=point_settings)
def test_jitter_interval(self): self._write_client.__del__() self._write_client = WriteApi(influxdb_client=self.influxdb_client, write_options=WriteOptions(batch_size=2, flush_interval=5_000, jitter_interval=3_000)) httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204) self._write_client.write("my-bucket", "my-org", ["h2o_feet,location=coyote_creek level\\ water_level=1.0 1", "h2o_feet,location=coyote_creek level\\ water_level=2.0 2"]) time.sleep(3) self.assertEqual(1, len(httpretty.httpretty.latest_requests)) self._write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek level\\ water_level=3.0 3") time.sleep(2) self.assertEqual(1, len(httpretty.httpretty.latest_requests)) time.sleep(6) self.assertEqual(2, len(httpretty.httpretty.latest_requests)) self.assertEqual("h2o_feet,location=coyote_creek level\\ water_level=3.0 3", httpretty.httpretty.latest_requests[1].parsed_body)
def test_default_tags(self): self._write_client.__del__() self.id_tag = "132-987-655" self.customer_tag = "California Miner" self._write_client = WriteApi(influxdb_client=self.influxdb_client, write_options=WriteOptions(batch_size=1), point_settings=PointSettings(**{"id": self.id_tag, "customer": self.customer_tag})) httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204) _point1 = {"measurement": "h2o_feet", "tags": {"location": "coyote_creek"}, "time": "2009-11-10T22:00:00Z", "fields": {"water_level": 1.0}} _point_list = [_point1] self._write_client.write("my-bucket", "my-org", _point_list) time.sleep(1) requests = httpretty.httpretty.latest_requests self.assertEqual(1, len(requests)) request = str(requests[0].body) self.assertNotEqual(-1, request.find('customer=California\\\\ Miner')) self.assertNotEqual(-1, request.find('id=132-987-655'))
def setUp(self) -> None: # https://github.com/gabrielfalcao/HTTPretty/issues/368 import warnings warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") warnings.filterwarnings("ignore", category=PendingDeprecationWarning, message="isAlive*") httpretty.enable() httpretty.reset() conf = influxdb_client.configuration.Configuration() conf.host = "http://localhost" conf.debug = False self.influxdb_client = influxdb_client.client.InfluxDBClient( url=conf.host, token="my-token") # self._api_client = influxdb_client.ApiClient(configuration=conf, header_name="Authorization", # header_value="Token my-token") write_options = WriteOptions(batch_size=2, flush_interval=5_000, retry_interval=3_000) self._write_client = WriteApi(influxdb_client=self.influxdb_client, write_options=write_options)
def write_api(self, write_options=WriteOptions()) -> WriteApi: """ Creates a Write API instance :param write_options: write api configuration :return: write api instance """ return WriteApi(influxdb_client=self, write_options=write_options)
def test_default(self): retry = WriteOptions().to_retry_strategy() self.assertEqual(retry.total, 5) self.assertEqual(retry.retry_interval, 5) self.assertEqual(retry.max_retry_time, 180) self.assertEqual(retry.max_retry_delay, 125) self.assertEqual(retry.exponential_base, 2) self.assertEqual(retry.allowed_methods, ["POST"])
def test_default(self): retry = WriteOptions().to_retry_strategy() self.assertEqual(retry.total, 3) self.assertEqual(retry.backoff_factor, 5) self.assertEqual(retry.jitter_interval, 0) self.assertEqual(retry.max_retry_delay, 180) self.assertEqual(retry.exponential_base, 5) self.assertEqual(retry.method_whitelist, ["POST"])
def write_api(self, write_options=WriteOptions(), point_settings=PointSettings()) -> WriteApi: """ Create a Write API instance. :param point_settings: :param write_options: write api configuration :return: write api instance """ return WriteApi(influxdb_client=self, write_options=write_options, point_settings=point_settings)
def __init__(self, influxdb_client: Union[InfluxDBClient, influxdb.InfluxDBClient], batch_size: int, flush_interval: int): self.client = influxdb_client if isinstance(influxdb_client, InfluxDBClient): self.write_api = self.client.write_api( WriteOptions(batch_size=batch_size, flush_interval=flush_interval))
def __init__(self, measurement_name: str, threads_count: int, seconds_count: int, line_protocols_count: int): Writer.__init__(self, measurement_name, threads_count, seconds_count, line_protocols_count) self.client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org", debug=False) self.write_api = self.client.write_api( WriteOptions(batch_size=50_000, flush_interval=10_000))
def __init__(self, config): self.logger = getLogger(self.__class__.__name__) self.org = config["influx"]["org"] self.bucket = config["influx"]["bucket"] client = InfluxDBClient(url=config["influx"]["url"], token=os.environ["INFLUX_TOKEN"]) write_options = WriteOptions( batch_size=config["influx"]["batch_size"], flush_interval=config["influx"]["interval"]) self.write_api = client.write_api(write_options=write_options)
def test_custom(self): retry = WriteOptions(max_retries=5, max_retry_delay=7500, retry_interval=500, jitter_interval=2000, exponential_base=2)\ .to_retry_strategy() self.assertEqual(retry.total, 5) self.assertEqual(retry.retry_interval, 0.5) self.assertEqual(retry.max_retry_delay, 7.5) self.assertEqual(retry.exponential_base, 2) self.assertEqual(retry.allowed_methods, ["POST"])
def test_custom(self): retry = WriteOptions(max_retries=5, max_retry_delay=7500, retry_interval=500, jitter_interval=2000, exponential_base=2)\ .to_retry_strategy() self.assertEqual(retry.total, 5) self.assertEqual(retry.backoff_factor, 0.5) self.assertEqual(retry.jitter_interval, 2) self.assertEqual(retry.max_retry_delay, 7.5) self.assertEqual(retry.exponential_base, 2) self.assertEqual(retry.method_whitelist, ["POST"])
def setup_influxdb(): client = InfluxDBClient(url=settings.influxdb.url, token=settings.influxdb.token) # TODO decide on Batching or Asynchronous # if high sample rate, go with batching, # if low, probs go async as it should be able to handle it write_api = client.write_api(write_options=WriteOptions( write_type=WriteType.batching, batch_size=100, )) return (client, write_api)
def __init__(self, url, org, bucket, token, size=50, flush_interval=10_000): self.logger = getLogger(self.__class__.__name__) self.org = org self.bucket = bucket client = InfluxDBClient(url=url, token=token) write_options = WriteOptions(batch_size=size, flush_interval=flush_interval) self.write_api = client.write_api(write_options=write_options)
def test_write_context_manager(self): with InfluxDBClient.from_env_properties(self.debug) as self.client: api_client = self.client.api_client with self.client.write_api(write_options=WriteOptions( write_type=WriteType.batching)) as write_api: write_api_test = write_api write_api.write(bucket="my-bucket", record=Point("h2o_feet").tag( "location", "coyote_creek").field( "level water_level", 5.0)) self.assertIsNotNone(write_api._subject) self.assertIsNotNone(write_api._disposable) self.assertIsNone(write_api_test._subject) self.assertIsNone(write_api_test._disposable) self.assertIsNotNone(self.client.api_client) self.assertIsNotNone( self.client.api_client.rest_client.pool_manager) self.assertIsNone(api_client._pool) self.assertIsNone(self.client.api_client)
def test_to_low_flush_interval(self): self._write_client.__del__() self._write_client = WriteApi(influxdb_client=self.influxdb_client, write_options=WriteOptions( batch_size=8, flush_interval=1, jitter_interval=1000)) httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204) for i in range(50): val_one = float(i) val_two = float(i) + 0.5 point_one = Point("OneMillis").tag("sensor", "sensor1").field( "PSI", val_one).time(time=i) point_two = Point("OneMillis").tag("sensor", "sensor2").field( "PSI", val_two).time(time=i) self._write_client.write("my-bucket", "my-org", [point_one, point_two]) time.sleep(0.1) self._write_client.__del__() _requests = httpretty.httpretty.latest_requests for _request in _requests: body = _request.parsed_body self.assertTrue(body, msg="Parsed body should be not empty " + str(_request)) httpretty.reset()
import time import datetime import pytz import os import yaml from influxdb_client import InfluxDBClient, Point from influxdb_client.client.write_api import SYNCHRONOUS, WriteOptions bucket = "sensors" client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") write_api = client.write_api( write_options=WriteOptions(batch_size=50_000, flush_interval=10_000)) #write_api = client.write_api(write_options=WriteOptions(batch_size=50, flush_interval=10)) #query_api = client.query_api() #delete_api = client.delete_api() """ Delete Data """ #start ="1970-01-01T00:00:00Z" #stop = "2021-05-25T00:00:00Z" #delete_api.delete(start, stop, '_measurement="my_measurement2"', bucket=bucket, org='my-org') def rainMessage(time, reading, nodeSettings): payload = emonSuite.PayloadRain() if (emonSuite.EmonSerial.ParseRainPayload(reading, payload)): try:
def write_api(self, write_options=WriteOptions(), point_settings=PointSettings(), **kwargs) -> WriteApi: """ Create a Write API instance. Example: .. code-block:: python from influxdb_client import InfluxDBClient from influxdb_client.client.write_api import SYNCHRONOUS # Initialize SYNCHRONOUS instance of WriteApi with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: write_api = client.write_api(write_options=SYNCHRONOUS) If you would like to use a **background batching**, you have to configure client like this: .. code-block:: python from influxdb_client import InfluxDBClient # Initialize background batching instance of WriteApi with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: with client.write_api() as write_api: pass There is also possibility to use callbacks to notify about state of background batches: .. code-block:: python from influxdb_client import InfluxDBClient from influxdb_client.client.exceptions import InfluxDBError class BatchingCallback(object): def success(self, conf: (str, str, str), data: str): print(f"Written batch: {conf}, data: {data}") def error(self, conf: (str, str, str), data: str, exception: InfluxDBError): print(f"Cannot write batch: {conf}, data: {data} due: {exception}") def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError): print(f"Retryable error occurs for batch: {conf}, data: {data} retry: {exception}") with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client: callback = BatchingCallback() with client.write_api(success_callback=callback.success, error_callback=callback.error, retry_callback=callback.retry) as write_api: pass :param write_options: Write API configuration :param point_settings: settings to store default tags :key success_callback: The callable ``callback`` to run after successfully writen a batch. The callable must accept two arguments: - `Tuple`: ``(bucket, organization, precision)`` - `str`: written data **[batching mode]** :key error_callback: The callable ``callback`` to run after unsuccessfully writen a batch. The callable must accept three arguments: - `Tuple`: ``(bucket, organization, precision)`` - `str`: written data - `Exception`: an occurred error **[batching mode]** :key retry_callback: The callable ``callback`` to run after retryable error occurred. The callable must accept three arguments: - `Tuple`: ``(bucket, organization, precision)`` - `str`: written data - `Exception`: an retryable error **[batching mode]** :return: write api instance """ return WriteApi(influxdb_client=self, write_options=write_options, point_settings=point_settings, **kwargs)