def __init__(
self,
bucket_name=BUCKET,
batch_size=LOG_BATCH_SIZE,
data_retention=3600,
):
self.organization = ORGANIZATION
self.client = InfluxDBClient(url=INFLUXDB_URL,
token=INFLUXDB_TOKEN,
org=self.organization)
self.batch_size = batch_size
self.bucket_name = bucket_name
self.write_api = self.client.write_api(write_options=WriteOptions(
batch_size=self.batch_size))
self.query_api = self.client.query_api()
self.buckets_api = self.client.buckets_api()
bucket = self.buckets_api.find_bucket_by_name(self.bucket_name)
if bucket is None:
logger.warning(f"Bucket {self.bucket_name!r} not found. "
f"Creating a bucket {self.bucket_name!r}.")
retention_rules = None
if data_retention is not None:
retention_rules = BucketRetentionRules(
type="expire", every_seconds=data_retention)
self.buckets_api.create_bucket(
bucket_name=self.bucket_name,
retention_rules=retention_rules,
org=self.organization,
)
def publishing_thread(self, url, token, org, bucket):
from influxdb_client import InfluxDBClient
bucket = bucket
client = InfluxDBClient(url=url, token=token, org=org)
from influxdb_client import WriteOptions
write_api = None
while True:
time.sleep(1)
if not write_api:
write_api = client.write_api(
write_options=WriteOptions(batch_size=100,
flush_interval=5_000,
jitter_interval=2_000,
retry_interval=2_000,
max_retries=3,
max_retry_delay=30_000,
exponential_base=2))
try:
buf = self.queue.copy()
self.queue.clear()
if len(buf) > 0:
write_api.write(bucket, org, buf)
time.sleep(5)
except:
write_api = None
time.sleep(10)
def init(self):
"""Init the connection to the InfluxDB server."""
if not self.export_enable:
return None
url = '{}://{}:{}'.format(self.protocol, self.host, self.port)
try:
client = InfluxDBClient(url=url,
enable_gzip=False,
org=self.org,
token=self.token)
except Exception as e:
logger.critical("Cannot connect to InfluxDB server '%s' (%s)" %
(url, e))
sys.exit(2)
else:
logger.info("Connected to InfluxDB server version {} ({})".format(
client.health().version,
client.health().message))
# Create the write client
write_client = client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10000,
jitter_interval=2000,
retry_interval=5000,
max_retries=5,
max_retry_delay=30000,
exponential_base=2))
return write_client
def write_to_influx(gesture_name, fields):
'''Write data to InfluxDB and return a Flask response'''
# Attempt to load the config_file file
try:
config_file = toml.load('config.toml')['influxdb']
except (FileNotFoundError, KeyError):
logging.error('Config file not found.')
influx_protocol = "http"
if config_file['tls']:
influx_protocol = "https"
# Initalize database collection
_client = InfluxDBClient(
url=f"{influx_protocol}://{config_file['host']}:{config_file['port']}",
verify_ssl=config_file['verify_tls'],
org=config_file['organization'],
token=config_file['token'])
# pylint: disable=line-too-long
_write_client = _client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
def __init__(self,
url: str,
token: str,
org: str,
data_bucket: str,
meta_bucket: str,
workers: int = cpu_count()):
super().__init__()
self.client = InfluxDBClient(url=url, token=token, org=org)
self.url = url
self.token = token
self.org = org
if not self.check_bucket_exists(data_bucket):
raise KeyError(f"Data bucket {data_bucket} as does not exist")
if not self.check_bucket_exists(meta_bucket):
raise KeyError(f"Meta bucket {meta_bucket} as does not exist")
self.data_bucket = data_bucket
self.meta_bucket = meta_bucket
# write with batch api with sane looking defaults
self.api = self.client.write_api(
write_options=WriteOptions(batch_size=200,
flush_interval=2000,
jitter_interval=100,
retry_interval=2000,
write_scheduler=ThreadPoolScheduler(
max_workers=workers)))
def test_large_amount_of_data(self):
_measurement_name = "data_frame_" + str(current_milli_time())
def _create_point(index) -> Point:
return Point(_measurement_name) \
.tag("deviceType", str(random.choice(['A', 'B']))) \
.tag("name", random.choice(['A', 'B'])) \
.field("uuid", random.randint(0, 10_000)) \
.field("co2", random.randint(0, 10_000)) \
.field("humid", random.randint(0, 10_000)) \
.field("lux", random.randint(0, 10_000)) \
.field("water", random.randint(0, 10_000)) \
.field("shine", random.randint(0, 10_000)) \
.field("temp", random.randint(0, 10_000)) \
.field("voc", random.randint(0, 10_000)) \
.time(time=(1583828781 + index), write_precision=WritePrecision.S)
data = rx.range(0, 2_000).pipe(ops.map(lambda index: _create_point(index)))
write_api = self.client.write_api(write_options=WriteOptions(batch_size=500))
write_api.write(org="my-org", bucket="my-bucket", record=data, write_precision=WritePrecision.S)
write_api.__del__()
query = 'from(bucket: "my-bucket")' \
'|> range(start: 2020-02-19T23:30:00Z, stop: now())' \
f'|> filter(fn: (r) => r._measurement == "{_measurement_name}")'
result = self.client.query_api().query_data_frame(org="my-org", query=query)
self.assertGreater(len(result), 1)
def main():
parse_row.progress = 0
url = "https://github.com/influxdata/influxdb-client-python/wiki/data/stock-prices-example.csv"
response = requests.get(url, stream=True)
data = rx \
.from_iterable(DictReader(response.iter_lines(decode_unicode=True))) \
.pipe(ops.map(lambda row: parse_row(row)))
client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org", debug=False)
write_api = client.write_api(write_options=WriteOptions(batch_size=50_000, flush_interval=10_000))
write_api.write(bucket="my-bucket", record=data)
write_api.__del__()
query = '''
from(bucket:"my-bucket")
|> range(start: 0, stop: now())
|> filter(fn: (r) => r._measurement == "financial-analysis")
|> filter(fn: (r) => r.symbol == "AAPL")
|> filter(fn: (r) => r._field == "close")
|> drop(columns: ["_start", "_stop", "table", "_field","_measurement"])
'''
result = client.query_api().query_data_frame(query=query)
print(result.head(100))
"""
Close client
"""
client.__del__()
def write_influx(self, name, df, tag_columns):
"""Helper function to write signal dataframes to InfluxDB"""
from influxdb_client import WriteOptions
if self.test == 0:
print("Please check your InfluxDB credentials")
return
_write_client = self.client.write_api(write_options=WriteOptions(
batch_size=5000,
flush_interval=1_000,
jitter_interval=2_000,
retry_interval=5_000,
))
_write_client.write(self.influx_bucket,
record=df,
data_frame_measurement_name=name,
data_frame_tag_columns=tag_columns)
if self.verbose:
print(
f"- SUCCESS: {len(df.index)} records of {name} written to InfluxDB\n\n"
)
_write_client.__del__()
def step001(self):
"""
Test general GlobalContainer-Functions
"""
pass
#s = engines.scaffold.addStock(self.gc, "IE00B6R52259")
#s = engines.scaffold.addStock(self.gc, "LU0323577923")
#engines.scaffold.getFondDistributions(self.gc, s)
try:
ts = datetime.datetime.strptime('2018-07-09 00:00:00',
'%Y-%m-%d %H:%M:%S')
engines.analysis.loadStock(self.gc, 'LU1681045370', ts)
sys.exit()
# https://github.com/influxdata/influxdb-client-python#queries
print(self.gc.influxClient)
query_api = self.gc.influxClient.query_api()
write_api = self.gc.influxClient.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
_now = datetime.datetime.now(UTC)
def config(config):
global influx2_debug
global influx2_client
global influx2_write_api
influx2_debug = int(config.get('debug', 0))
org = config.get('org', "my-org")
url = config.get('url', "http://localhost:8086")
token = config.get('token', "my-token")
# create connection
influx2_client.close()
if influx2_debug > 0:
influx2_client = InfluxDBClient(url=url,
token=token,
org=org,
debug=True)
else:
influx2_client = InfluxDBClient(url=url,
token=token,
org=org,
debug=False)
influx2_write_api.close()
influx2_write_api = influx2_client.write_api(
write_options=WriteOptions(batch_size=200,
flush_interval=120_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
def __init__(self):
self.client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
debug=False)
self.write_api = self.client.write_api(
write_options=WriteOptions(write_type=WriteType.batching,
batch_size=50_000,
flush_interval=10_000))
def init_connection():
client = InfluxDBClient(url="http://localhost:8086", token=token, org=org)
write_client = client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
def setUp(self) -> None:
super().setUp()
self.influxDb_client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
debug=False)
self.write_options = WriteOptions(batch_size=10_000,
flush_interval=5_000,
retry_interval=3_000)
self._write_client = WriteApi(influxdb_client=self.influxDb_client,
write_options=self.write_options)
def __init__(self, queue):
multiprocessing.Process.__init__(self)
self.queue = queue
self.client = InfluxDBClient(url="http://localhost:9999",
token="my-token",
org="my-org",
debug=False)
self.write_api = self.client.write_api(
write_options=WriteOptions(write_type=WriteType.batching,
batch_size=50_000,
flush_interval=10_000))
def start(self):
""" Start the submission thread """
self._running = True
self._influx = InfluxDBClient(url=self._config['url'],
org=self._config['org'],
token=self._config['token'],
enable_gzip=True)
opts = WriteOptions(batch_size=self._batch_size,
flush_interval=self._poll_interval * 1000,
jitter_interval=5000)
self._iwrite = self._influx.write_api(write_options=opts)
self._car.register_data(self.data_callback)
def import_data(config, data):
points = []
for d in data:
point = create_point(d)
points.append(point)
client = InfluxDBClient(url=config['influxdb_url'], token=config['influxdb_token'], org=config['influxdb_org'], debug=False)
write_api = client.write_api(write_options=WriteOptions(batch_size=100, flush_interval=50))
write_api.write(bucket=config['influxdb_bucket'], record=points)
write_api.close()
print("Imported {} records".format(len(points)))
def _influxdb_write(entry, url, token, org, bucket, debug, verify_ssl,
convert):
with InfluxDBClient(url=url,
token=token,
org=org,
debug=debug,
verify_ssl=verify_ssl) as _client:
with _client.write_api(write_options=WriteOptions(
batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=2,
max_retry_delay=30_000,
exponential_base=2)) as _write_client:
for entry_id in entry.influxdb_event_buffer:
for eventname in entry.influxdb_event_buffer[entry_id]:
for k in entry.influxdb_event_buffer[entry_id][eventname]:
for d in entry.influxdb_event_buffer[entry_id][
eventname][k]:
# Keep only eventdata of really published messages
if d['time'] > 0:
#params = d['changed_params'] if d['changed_params'] else d['params']
params = d['params']
# Keep only primitive values and convert values
params = {
x: _influxdb_type_conversion(
params[x], x, d['params'])
if convert else params[x]
for x in params
if x != "temporary" and (":" not in x)
and isinstance(params[x], (int, str, bool,
float))
}
# Use keys and "temporary" as tags
tags = d['keys']
if 'temporary' in d['params'] and d['params'][
'temporary']:
tags['temporary'] = True
if params:
_write_client.write(
bucket, org, {
"measurement":
entry_id + "." + eventname,
"tags":
tags,
"fields":
params,
"time":
datetime.datetime.utcfromtimestamp(
d['time']),
})
def writetodb(data_points):
client = InfluxDBClient(url=influx_url,
token=influx_token,
org=influx_org)
try:
data_write = client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
data_write.write(my_bucket, influx_org, data_points)
def open_con(cls):
load_dotenv()
cls.token = os.getenv('INFLUXDB_V2_TOKEN')
cls.org = os.getenv('INFLUXDB_V2_ORG')
cls.bucket = os.getenv('INFLUXDB_V2_BUCKET')
cls.url = "http://influxdb:8086"
cls.client = InfluxDBClient(url=cls.url, token=cls.token, org=cls.org)
print('Connected')
cls.write_api = cls.client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
def connectInfluxDatabase(self):
try:
# prepare database
self.logger.debug(
f'Connecting to Influx with: Host:{self.influx_host}, Port: {self.influx_port}, User: {self.influx_user}, DB: {self.influx_db}'
)
if (self.influx_version == 1):
pass
self.influxClient = DataFrameClient(self.influx_host,
self.influx_port,
self.influx_user,
self.influx_pwd,
self.influx_db)
elif (self.influx_version == 2):
retries = WritesRetry(total=20,
backoff_factor=1,
exponential_base=1)
self.influxClient = InfluxDBClient(
url=f"http://{self.influx_host}:{self.influx_port}",
token=self.influx_token,
org=self.influx_org,
retries=retries,
timeout=180_000)
self.influx_query_api = self.influxClient.query_api()
self.influx_write_api = self.influxClient.write_api(
write_options=WriteOptions(
batch_size=500,
write_type=WriteType.synchronous,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=30_000,
max_retries=25,
max_retry_delay=60_000,
exponential_base=2))
#self.influx_write_api = self.influxClient.write_api(write_options=SYNCHRONOUS)
except Exception as e:
self.logger.exception('Crash!', exc_info=e)
sys.exit(99)
def configure() -> None:
"""
Retrieve or refresh a configuration from IoT Center.
Successful configuration is set as a global IOT_CONFIGURATION dictionary with following properties:
* id
* influx_url
* influx_org
* influx_token
* influx_bucket
* configuration_refresh
* default_lon
* default_lat
* measurement_interval
"""
global config
global config_received
global influxdb_client
global write_api
# Check freshness of configuration
if config_received and (datetime.utcnow() - config_received).total_seconds(
) < config['configuration_refresh']:
return
iot_center_url = os.getenv("IOT_CENTER_URL", "http://localhost:5000")
iot_device_id = os.getenv("IOT_DEVICE_ID")
# Request to configuration
config_fresh = fetch_json(f'{iot_center_url}/api/env/{iot_device_id}')
# New or changed configuration
if not config and config_fresh != config:
config = config_fresh
config_received = datetime.utcnow()
influxdb_client = InfluxDBClient(url=config['influx_url'],
token=config['influx_token'],
org=config['influx_org'])
write_api = influxdb_client.write_api(write_options=WriteOptions(
batch_size=1))
print(
f'Received configuration: {json.dumps(config, indent=4, sort_keys=False)}'
)
def __influxInit(self):
if self.__options.verbosity > 0:
print("initializing influx configuration", flush=True)
# the only way i could find to see how many points are getting flushed is to remap some internal influx functions
# _newHttp will ack points in rabbit
WriteApi._origHttp = WriteApi._http
WriteApi._http = _newHttp
writeOptions = WriteOptions(batch_size=self.__points.maxPoints,
flush_interval=self.__points.flushInterval)
self.__influxClient = InfluxDBClient(url=self.__options.influx2URL,
token=self.__options.influx2Token,
org=self.__options.influx2Org,
debug=self.__options.debug)
self.__influx = self.__influxClient.write_api(
write_options=writeOptions)
self.__influx.main = self
def run(self):
"""Initialize ``InfluxDBClient`` and waits for data to writes into InfluxDB."""
# Initialize Client and Write API
self.client = InfluxDBClient(**self.kwargs)
self.write_api = self.client.write_api(
write_options=self.kwargs.get('write_options', WriteOptions()),
success_callback=self.kwargs.get('success_callback',
_success_callback),
error_callback=self.kwargs.get('error_callback', _error_callback),
retry_callback=self.kwargs.get('retry_callback', _retry_callback))
# Infinite loop - until poison pill
while True:
next_record = self.queue_.get()
if type(next_record) is _PoisonPill:
# Poison pill means break the loop
self.terminate()
self.queue_.task_done()
break
self.write_api.write(**next_record)
self.queue_.task_done()
def influx_client_write_data_frame(host='http://localhost:8086',
df=None,
df_time_index_name='',
database='',
measurement_name='',
tag_columns=[]):
""" 保存数据
df: 数据 DataFrame
df_time_index_name: 数据列（InfluxDB只能使用时间作为主键（唯一标志），这里需要把这列设置为 Index）
database: 库名（这里需要现在 InfluxDB 内创建好数据库）
measurement_name: 表名
tag_columns: 索引
"""
client = InfluxDBClient(url=host, token="", org="", debug=False)
write_client = client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
elif os.path.exists('config.yaml'):
import yaml
with open('config.yaml', encoding='utf-8') as config_file:
C = None
for c in yaml.load_all(config_file, Loader=yaml.SafeLoader):
C = c
else:
raise Exception('No config found')
Influx = InfluxDBClient(url=C['influxdb']['url'],
org=C['influxdb']['org'],
token=C['influxdb']['token'],
enable_gzip=True)
opts = WriteOptions(batch_size=10000,
flush_interval=60000,
jitter_interval=5000)
write_api = Influx.write_api(write_options=opts)
##############################################################################
if __name__ == "__main__":
ser = Serial("/dev/ttyUSB0")
while True:
line = ser.readline()
if line[:6] == b'^RSSI:':
now = time.time()
data = {
'Strength': int(line[6:]),
}
# %%
cp = forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper', 'measurement']].copy()
lines = [
str(cp["measurement"][d]) + ",type=forecast" + " " + "yhat=" +
str(cp["yhat"][d]) + "," + "yhat_lower=" + str(cp["yhat_lower"][d]) + "," +
"yhat_upper=" + str(cp["yhat_upper"][d]) + " " +
str(int(time.mktime(cp['ds'][d].timetuple()))) + "000000000"
for d in range(len(cp))
]
# %%
from influxdb_client import InfluxDBClient, Point, WriteOptions
from influxdb_client.client.write_api import SYNCHRONOUS
_write_client = client.write_api(
write_options=WriteOptions(batch_size=1000,
flush_interval=10000,
jitter_interval=2000,
retry_interval=5000))
_write_client.write(bucket, org, lines)
lines[0:10]
# %% [markdown]
# To close client:
# %%
_write_client.__del__()
client.__del__()
import pyarrow.parquet as pq
from influxdb_client import InfluxDBClient, WriteOptions
with InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
timeout=0,
debug=False) as client:
"""
You can download NYC TLC Trip Record Data parquet file from https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
"""
table = pq.read_table('fhvhv_tripdata_2022-01.parquet')
with client.write_api(write_options=WriteOptions(
batch_size=50_000)) as write_api:
dataframe = table.to_pandas()
"""
Keep only interesting columns
"""
keep_df = dataframe[[
'dispatching_base_num', "PULocationID", "DOLocationID",
"pickup_datetime", "dropoff_datetime", "shared_request_flag"
]]
print(keep_df.tail().to_string())
write_api.write(bucket="my-bucket",
record=keep_df,
data_frame_measurement_name="taxi-trip-data",
data_frame_tag_columns=[
'dispatching_base_num', "shared_request_flag"
"""
Converts vix-daily.csv into sequence of data point
"""
data = rx \
.from_iterable(DictReader(open('vix-daily.csv', 'r'))) \
.pipe(ops.map(lambda row: parse_row(row)))
client = InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
debug=True)
"""
Create client that writes data in batches with 50_000 items.
"""
write_api = client.write_api(
write_options=WriteOptions(batch_size=50_000, flush_interval=10_000))
"""
Write data into InfluxDB
"""
write_api.write(bucket="my-bucket", record=data)
write_api.close()
"""
Querying max value of CBOE Volatility Index
"""
query = 'from(bucket:"my-bucket")' \
' |> range(start: 0, stop: now())' \
' |> filter(fn: (r) => r._measurement == "financial-analysis")' \
' |> max()'
result = client.query_api().query(query=query)
"""
Processing results
"""
Converts vix-daily.csv into sequence of data point
"""
data = rx \
.from_iterable(DictReader(open('vix-daily.csv', 'r'))) \
.pipe(ops.map(lambda row: parse_row(row)))
with InfluxDBClient(url="http://localhost:8086",
token="my-token",
org="my-org",
debug=True) as client:
"""
Create client that writes data in batches with 50_000 items.
"""
with client.write_api(write_options=WriteOptions(
batch_size=50_000, flush_interval=10_000)) as write_api:
"""
Write data into InfluxDB
"""
write_api.write(bucket="my-bucket", record=data)
"""
Querying max value of CBOE Volatility Index
"""
query = 'from(bucket:"my-bucket")' \
' |> range(start: 0, stop: now())' \
' |> filter(fn: (r) => r._measurement == "financial-analysis")' \
' |> max()'
result = client.query_api().query(query=query)
"""
Processing results
"""
import socket
return 'iot_sensor,hostname={},type=temperature value={}'.format(
socket.gethostname(), temperature)
"""
Read temperature every minute; distinct_until_changed - produce only if temperature change
"""
data = rx \
.interval(period=timedelta(seconds=60)) \
.pipe(ops.map(lambda t: sensor_temperature()),
ops.distinct_until_changed(),
ops.map(lambda temperature: line_protocol(temperature)))
_db_client = InfluxDBClient(url="http://localhost:9999",
token="my-token",
org="my-org",
debug=True)
"""
Create client that writes data into InfluxDB
"""
_write_api = _db_client.write_api(write_options=WriteOptions(batch_size=1))
_write_api.write(org="my-org", bucket="my-bucket", record=data)
"""
Call after terminate a script
"""
atexit.register(on_exit, _db_client, _write_api)
input()
