def __init__(self, url="http://localhost:8086", settingsPath="./emon_config.yml",batchProcess=True):
self.bucket = "sensors"
self.client = InfluxDBClient(url, token="my-token", org="my-org")
if(batchProcess):
self.write_api = self.client.write_api(write_options=WriteOptions(batch_size=50_000, flush_interval=10_000))
else:
self.write_api = self.client.write_api(write_options=WriteOptions(batch_size=50, flush_interval=10))
self.lineNumber = -1
settingsFile = open(settingsPath, 'r')
self.settings = yaml.full_load(settingsFile)
self.dispatch = {
'rain' : self.rainMessage,
'temp' : self.temperatureMessage,
'disp' : self.dispMessage,
'pulse': self.pulseMessage,
'hws' : self.HWSMessage,
'wtr' : self.waterMessage,
'scl' : self.scaleMessage,
'bat' : self.batteryMessage,
'inv' : self.inverterMessage,
'bee' : self.beeMessage,
'air' : self.airMessage,
'leaf' : self.leafMessage,
'other': self.otherMessage
}
def write_api(
self, write_options=WriteOptions(), point_settings=PointSettings()
) -> WriteApi:
"""
Create a Write API instance.
:param point_settings:
:param write_options: write api configuration
:return: write api instance
"""
#WriteApi 객체를 생성해서 리턴하는것임. 인자로 받는 write_options와 point_settings 모두 클래스네. 원형을 하나만 찾아보자!
"""
class WriteOptions(object):
def __init__(self, write_type: WriteType = WriteType.batching,
batch_size=1_000, flush_interval=1_000,
jitter_interval=0,
retry_interval=5_000,
max_retries=3,
max_retry_delay=180_000,
exponential_base=5,
write_scheduler=ThreadPoolScheduler(max_workers=1)) -> None:
self.write_type = write_type
self.batch_size = batch_size
self.flush_interval = flush_interval
self.jitter_interval = jitter_interval
self.retry_interval = retry_interval
self.max_retries = max_retries
self.max_retry_delay = max_retry_delay
self.exponential_base = exponential_base
self.write_scheduler = write_scheduler
"""
# 보면 위 객체는 그냥 값들만 담고있음. 그니까 어떻게보면 javascript식 객체생성을 하는 느낌이네.
return WriteApi(influxdb_client=self,
write_options=write_options,
point_settings=point_settings)
def test_jitter_interval(self):
self._write_client.__del__()
self._write_client = WriteApi(influxdb_client=self.influxdb_client,
write_options=WriteOptions(batch_size=2, flush_interval=5_000,
jitter_interval=3_000))
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204)
self._write_client.write("my-bucket", "my-org",
["h2o_feet,location=coyote_creek level\\ water_level=1.0 1",
"h2o_feet,location=coyote_creek level\\ water_level=2.0 2"])
time.sleep(3)
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
self._write_client.write("my-bucket", "my-org", "h2o_feet,location=coyote_creek level\\ water_level=3.0 3")
time.sleep(2)
self.assertEqual(1, len(httpretty.httpretty.latest_requests))
time.sleep(6)
self.assertEqual(2, len(httpretty.httpretty.latest_requests))
self.assertEqual("h2o_feet,location=coyote_creek level\\ water_level=3.0 3",
httpretty.httpretty.latest_requests[1].parsed_body)
def test_default_tags(self):
self._write_client.__del__()
self.id_tag = "132-987-655"
self.customer_tag = "California Miner"
self._write_client = WriteApi(influxdb_client=self.influxdb_client,
write_options=WriteOptions(batch_size=1),
point_settings=PointSettings(**{"id": self.id_tag,
"customer": self.customer_tag}))
httpretty.register_uri(httpretty.POST, uri="http://localhost/api/v2/write", status=204)
_point1 = {"measurement": "h2o_feet", "tags": {"location": "coyote_creek"},
"time": "2009-11-10T22:00:00Z", "fields": {"water_level": 1.0}}
_point_list = [_point1]
self._write_client.write("my-bucket", "my-org", _point_list)
time.sleep(1)
requests = httpretty.httpretty.latest_requests
self.assertEqual(1, len(requests))
request = str(requests[0].body)
self.assertNotEqual(-1, request.find('customer=California\\\\ Miner'))
self.assertNotEqual(-1, request.find('id=132-987-655'))
def setUp(self) -> None:
# https://github.com/gabrielfalcao/HTTPretty/issues/368
import warnings
warnings.filterwarnings("ignore",
category=ResourceWarning,
message="unclosed.*")
warnings.filterwarnings("ignore",
category=PendingDeprecationWarning,
message="isAlive*")
httpretty.enable()
httpretty.reset()
conf = influxdb_client.configuration.Configuration()
conf.host = "http://localhost"
conf.debug = False
self.influxdb_client = influxdb_client.client.InfluxDBClient(
url=conf.host, token="my-token")
# self._api_client = influxdb_client.ApiClient(configuration=conf, header_name="Authorization",
# header_value="Token my-token")
write_options = WriteOptions(batch_size=2,
flush_interval=5_000,
retry_interval=3_000)
self._write_client = WriteApi(influxdb_client=self.influxdb_client,
write_options=write_options)
def write_api(self, write_options=WriteOptions()) -> WriteApi:
"""
Creates a Write API instance
:param write_options: write api configuration
:return: write api instance
"""
return WriteApi(influxdb_client=self, write_options=write_options)
def test_default(self):
retry = WriteOptions().to_retry_strategy()
self.assertEqual(retry.total, 5)
self.assertEqual(retry.retry_interval, 5)
self.assertEqual(retry.max_retry_time, 180)
self.assertEqual(retry.max_retry_delay, 125)
self.assertEqual(retry.exponential_base, 2)
self.assertEqual(retry.allowed_methods, ["POST"])
def test_default(self):
retry = WriteOptions().to_retry_strategy()
self.assertEqual(retry.total, 3)
self.assertEqual(retry.backoff_factor, 5)
self.assertEqual(retry.jitter_interval, 0)
self.assertEqual(retry.max_retry_delay, 180)
self.assertEqual(retry.exponential_base, 5)
self.assertEqual(retry.method_whitelist, ["POST"])
def write_api(self, write_options=WriteOptions(), point_settings=PointSettings()) -> WriteApi:
"""
Create a Write API instance.
:param point_settings:
:param write_options: write api configuration
:return: write api instance
"""
return WriteApi(influxdb_client=self, write_options=write_options, point_settings=point_settings)
def __init__(self, influxdb_client: Union[InfluxDBClient,
influxdb.InfluxDBClient],
batch_size: int, flush_interval: int):
self.client = influxdb_client
if isinstance(influxdb_client, InfluxDBClient):
self.write_api = self.client.write_api(
WriteOptions(batch_size=batch_size,
flush_interval=flush_interval))
def __init__(self, measurement_name: str, threads_count: int,
seconds_count: int, line_protocols_count: int):
Writer.__init__(self, measurement_name, threads_count, seconds_count,
line_protocols_count)
self.client = InfluxDBClient(url="http://localhost:9999",
token="my-token",
org="my-org",
debug=False)
self.write_api = self.client.write_api(
WriteOptions(batch_size=50_000, flush_interval=10_000))
def __init__(self, config):
self.logger = getLogger(self.__class__.__name__)
self.org = config["influx"]["org"]
self.bucket = config["influx"]["bucket"]
client = InfluxDBClient(url=config["influx"]["url"],
token=os.environ["INFLUX_TOKEN"])
write_options = WriteOptions(
batch_size=config["influx"]["batch_size"],
flush_interval=config["influx"]["interval"])
self.write_api = client.write_api(write_options=write_options)
def test_custom(self):
retry = WriteOptions(max_retries=5, max_retry_delay=7500,
retry_interval=500, jitter_interval=2000,
exponential_base=2)\
.to_retry_strategy()
self.assertEqual(retry.total, 5)
self.assertEqual(retry.retry_interval, 0.5)
self.assertEqual(retry.max_retry_delay, 7.5)
self.assertEqual(retry.exponential_base, 2)
self.assertEqual(retry.allowed_methods, ["POST"])
def test_custom(self):
retry = WriteOptions(max_retries=5, max_retry_delay=7500,
retry_interval=500, jitter_interval=2000,
exponential_base=2)\
.to_retry_strategy()
self.assertEqual(retry.total, 5)
self.assertEqual(retry.backoff_factor, 0.5)
self.assertEqual(retry.jitter_interval, 2)
self.assertEqual(retry.max_retry_delay, 7.5)
self.assertEqual(retry.exponential_base, 2)
self.assertEqual(retry.method_whitelist, ["POST"])
def setup_influxdb():
client = InfluxDBClient(url=settings.influxdb.url, token=settings.influxdb.token)
# TODO decide on Batching or Asynchronous
# if high sample rate, go with batching,
# if low, probs go async as it should be able to handle it
write_api = client.write_api(write_options=WriteOptions(
write_type=WriteType.batching,
batch_size=100,
))
return (client, write_api)
def __init__(self,
url,
org,
bucket,
token,
size=50,
flush_interval=10_000):
self.logger = getLogger(self.__class__.__name__)
self.org = org
self.bucket = bucket
client = InfluxDBClient(url=url, token=token)
write_options = WriteOptions(batch_size=size,
flush_interval=flush_interval)
self.write_api = client.write_api(write_options=write_options)
def test_write_context_manager(self):
with InfluxDBClient.from_env_properties(self.debug) as self.client:
api_client = self.client.api_client
with self.client.write_api(write_options=WriteOptions(
write_type=WriteType.batching)) as write_api:
write_api_test = write_api
write_api.write(bucket="my-bucket",
record=Point("h2o_feet").tag(
"location", "coyote_creek").field(
"level water_level", 5.0))
self.assertIsNotNone(write_api._subject)
self.assertIsNotNone(write_api._disposable)
self.assertIsNone(write_api_test._subject)
self.assertIsNone(write_api_test._disposable)
self.assertIsNotNone(self.client.api_client)
self.assertIsNotNone(
self.client.api_client.rest_client.pool_manager)
self.assertIsNone(api_client._pool)
self.assertIsNone(self.client.api_client)
def test_to_low_flush_interval(self):
self._write_client.__del__()
self._write_client = WriteApi(influxdb_client=self.influxdb_client,
write_options=WriteOptions(
batch_size=8,
flush_interval=1,
jitter_interval=1000))
httpretty.register_uri(httpretty.POST,
uri="http://localhost/api/v2/write",
status=204)
for i in range(50):
val_one = float(i)
val_two = float(i) + 0.5
point_one = Point("OneMillis").tag("sensor", "sensor1").field(
"PSI", val_one).time(time=i)
point_two = Point("OneMillis").tag("sensor", "sensor2").field(
"PSI", val_two).time(time=i)
self._write_client.write("my-bucket", "my-org",
[point_one, point_two])
time.sleep(0.1)
self._write_client.__del__()
_requests = httpretty.httpretty.latest_requests
for _request in _requests:
body = _request.parsed_body
self.assertTrue(body,
msg="Parsed body should be not empty " +
str(_request))
httpretty.reset()
import time
import datetime
import pytz
import os
import yaml
from influxdb_client import InfluxDBClient, Point
from influxdb_client.client.write_api import SYNCHRONOUS, WriteOptions
bucket = "sensors"
client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org")
write_api = client.write_api(
write_options=WriteOptions(batch_size=50_000, flush_interval=10_000))
#write_api = client.write_api(write_options=WriteOptions(batch_size=50, flush_interval=10))
#query_api = client.query_api()
#delete_api = client.delete_api()
"""
Delete Data
"""
#start ="1970-01-01T00:00:00Z"
#stop = "2021-05-25T00:00:00Z"
#delete_api.delete(start, stop, '_measurement="my_measurement2"', bucket=bucket, org='my-org')
def rainMessage(time, reading, nodeSettings):
payload = emonSuite.PayloadRain()
if (emonSuite.EmonSerial.ParseRainPayload(reading, payload)):
try:
def write_api(self,
write_options=WriteOptions(),
point_settings=PointSettings(),
**kwargs) -> WriteApi:
"""
Create a Write API instance.
Example:
.. code-block:: python
from influxdb_client import InfluxDBClient
from influxdb_client.client.write_api import SYNCHRONOUS
# Initialize SYNCHRONOUS instance of WriteApi
with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client:
write_api = client.write_api(write_options=SYNCHRONOUS)
If you would like to use a **background batching**, you have to configure client like this:
.. code-block:: python
from influxdb_client import InfluxDBClient
# Initialize background batching instance of WriteApi
with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client:
with client.write_api() as write_api:
pass
There is also possibility to use callbacks to notify about state of background batches:
.. code-block:: python
from influxdb_client import InfluxDBClient
from influxdb_client.client.exceptions import InfluxDBError
class BatchingCallback(object):
def success(self, conf: (str, str, str), data: str):
print(f"Written batch: {conf}, data: {data}")
def error(self, conf: (str, str, str), data: str, exception: InfluxDBError):
print(f"Cannot write batch: {conf}, data: {data} due: {exception}")
def retry(self, conf: (str, str, str), data: str, exception: InfluxDBError):
print(f"Retryable error occurs for batch: {conf}, data: {data} retry: {exception}")
with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org") as client:
callback = BatchingCallback()
with client.write_api(success_callback=callback.success,
error_callback=callback.error,
retry_callback=callback.retry) as write_api:
pass
:param write_options: Write API configuration
:param point_settings: settings to store default tags
:key success_callback: The callable ``callback`` to run after successfully writen a batch.
The callable must accept two arguments:
- `Tuple`: ``(bucket, organization, precision)``
- `str`: written data
**[batching mode]**
:key error_callback: The callable ``callback`` to run after unsuccessfully writen a batch.
The callable must accept three arguments:
- `Tuple`: ``(bucket, organization, precision)``
- `str`: written data
- `Exception`: an occurred error
**[batching mode]**
:key retry_callback: The callable ``callback`` to run after retryable error occurred.
The callable must accept three arguments:
- `Tuple`: ``(bucket, organization, precision)``
- `str`: written data
- `Exception`: an retryable error
**[batching mode]**
:return: write api instance
"""
return WriteApi(influxdb_client=self,
write_options=write_options,
point_settings=point_settings,
**kwargs)
