def __init__( self, bucket_name=BUCKET, batch_size=LOG_BATCH_SIZE, data_retention=3600, ): self.organization = ORGANIZATION self.client = InfluxDBClient(url=INFLUXDB_URL, token=INFLUXDB_TOKEN, org=self.organization) self.batch_size = batch_size self.bucket_name = bucket_name self.write_api = self.client.write_api(write_options=WriteOptions( batch_size=self.batch_size)) self.query_api = self.client.query_api() self.buckets_api = self.client.buckets_api() bucket = self.buckets_api.find_bucket_by_name(self.bucket_name) if bucket is None: logger.warning(f"Bucket {self.bucket_name!r} not found. " f"Creating a bucket {self.bucket_name!r}.") retention_rules = None if data_retention is not None: retention_rules = BucketRetentionRules( type="expire", every_seconds=data_retention) self.buckets_api.create_bucket( bucket_name=self.bucket_name, retention_rules=retention_rules, org=self.organization, )
def publishing_thread(self, url, token, org, bucket): from influxdb_client import InfluxDBClient bucket = bucket client = InfluxDBClient(url=url, token=token, org=org) from influxdb_client import WriteOptions write_api = None while True: time.sleep(1) if not write_api: write_api = client.write_api( write_options=WriteOptions(batch_size=100, flush_interval=5_000, jitter_interval=2_000, retry_interval=2_000, max_retries=3, max_retry_delay=30_000, exponential_base=2)) try: buf = self.queue.copy() self.queue.clear() if len(buf) > 0: write_api.write(bucket, org, buf) time.sleep(5) except: write_api = None time.sleep(10)
def init(self): """Init the connection to the InfluxDB server.""" if not self.export_enable: return None url = '{}://{}:{}'.format(self.protocol, self.host, self.port) try: client = InfluxDBClient(url=url, enable_gzip=False, org=self.org, token=self.token) except Exception as e: logger.critical("Cannot connect to InfluxDB server '%s' (%s)" % (url, e)) sys.exit(2) else: logger.info("Connected to InfluxDB server version {} ({})".format( client.health().version, client.health().message)) # Create the write client write_client = client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10000, jitter_interval=2000, retry_interval=5000, max_retries=5, max_retry_delay=30000, exponential_base=2)) return write_client
def write_to_influx(gesture_name, fields): '''Write data to InfluxDB and return a Flask response''' # Attempt to load the config_file file try: config_file = toml.load('config.toml')['influxdb'] except (FileNotFoundError, KeyError): logging.error('Config file not found.') influx_protocol = "http" if config_file['tls']: influx_protocol = "https" # Initalize database collection _client = InfluxDBClient( url=f"{influx_protocol}://{config_file['host']}:{config_file['port']}", verify_ssl=config_file['verify_tls'], org=config_file['organization'], token=config_file['token']) # pylint: disable=line-too-long _write_client = _client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2))
def __init__(self, url: str, token: str, org: str, data_bucket: str, meta_bucket: str, workers: int = cpu_count()): super().__init__() self.client = InfluxDBClient(url=url, token=token, org=org) self.url = url self.token = token self.org = org if not self.check_bucket_exists(data_bucket): raise KeyError(f"Data bucket {data_bucket} as does not exist") if not self.check_bucket_exists(meta_bucket): raise KeyError(f"Meta bucket {meta_bucket} as does not exist") self.data_bucket = data_bucket self.meta_bucket = meta_bucket # write with batch api with sane looking defaults self.api = self.client.write_api( write_options=WriteOptions(batch_size=200, flush_interval=2000, jitter_interval=100, retry_interval=2000, write_scheduler=ThreadPoolScheduler( max_workers=workers)))
def test_large_amount_of_data(self): _measurement_name = "data_frame_" + str(current_milli_time()) def _create_point(index) -> Point: return Point(_measurement_name) \ .tag("deviceType", str(random.choice(['A', 'B']))) \ .tag("name", random.choice(['A', 'B'])) \ .field("uuid", random.randint(0, 10_000)) \ .field("co2", random.randint(0, 10_000)) \ .field("humid", random.randint(0, 10_000)) \ .field("lux", random.randint(0, 10_000)) \ .field("water", random.randint(0, 10_000)) \ .field("shine", random.randint(0, 10_000)) \ .field("temp", random.randint(0, 10_000)) \ .field("voc", random.randint(0, 10_000)) \ .time(time=(1583828781 + index), write_precision=WritePrecision.S) data = rx.range(0, 2_000).pipe(ops.map(lambda index: _create_point(index))) write_api = self.client.write_api(write_options=WriteOptions(batch_size=500)) write_api.write(org="my-org", bucket="my-bucket", record=data, write_precision=WritePrecision.S) write_api.__del__() query = 'from(bucket: "my-bucket")' \ '|> range(start: 2020-02-19T23:30:00Z, stop: now())' \ f'|> filter(fn: (r) => r._measurement == "{_measurement_name}")' result = self.client.query_api().query_data_frame(org="my-org", query=query) self.assertGreater(len(result), 1)
def main(): parse_row.progress = 0 url = "https://github.com/influxdata/influxdb-client-python/wiki/data/stock-prices-example.csv" response = requests.get(url, stream=True) data = rx \ .from_iterable(DictReader(response.iter_lines(decode_unicode=True))) \ .pipe(ops.map(lambda row: parse_row(row))) client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org", debug=False) write_api = client.write_api(write_options=WriteOptions(batch_size=50_000, flush_interval=10_000)) write_api.write(bucket="my-bucket", record=data) write_api.__del__() query = ''' from(bucket:"my-bucket") |> range(start: 0, stop: now()) |> filter(fn: (r) => r._measurement == "financial-analysis") |> filter(fn: (r) => r.symbol == "AAPL") |> filter(fn: (r) => r._field == "close") |> drop(columns: ["_start", "_stop", "table", "_field","_measurement"]) ''' result = client.query_api().query_data_frame(query=query) print(result.head(100)) """ Close client """ client.__del__()
def write_influx(self, name, df, tag_columns): """Helper function to write signal dataframes to InfluxDB""" from influxdb_client import WriteOptions if self.test == 0: print("Please check your InfluxDB credentials") return _write_client = self.client.write_api(write_options=WriteOptions( batch_size=5000, flush_interval=1_000, jitter_interval=2_000, retry_interval=5_000, )) _write_client.write(self.influx_bucket, record=df, data_frame_measurement_name=name, data_frame_tag_columns=tag_columns) if self.verbose: print( f"- SUCCESS: {len(df.index)} records of {name} written to InfluxDB\n\n" ) _write_client.__del__()
def step001(self): """ Test general GlobalContainer-Functions """ pass #s = engines.scaffold.addStock(self.gc, "IE00B6R52259") #s = engines.scaffold.addStock(self.gc, "LU0323577923") #engines.scaffold.getFondDistributions(self.gc, s) try: ts = datetime.datetime.strptime('2018-07-09 00:00:00', '%Y-%m-%d %H:%M:%S') engines.analysis.loadStock(self.gc, 'LU1681045370', ts) sys.exit() # https://github.com/influxdata/influxdb-client-python#queries print(self.gc.influxClient) query_api = self.gc.influxClient.query_api() write_api = self.gc.influxClient.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2)) _now = datetime.datetime.now(UTC)
def config(config): global influx2_debug global influx2_client global influx2_write_api influx2_debug = int(config.get('debug', 0)) org = config.get('org', "my-org") url = config.get('url', "http://localhost:8086") token = config.get('token', "my-token") # create connection influx2_client.close() if influx2_debug > 0: influx2_client = InfluxDBClient(url=url, token=token, org=org, debug=True) else: influx2_client = InfluxDBClient(url=url, token=token, org=org, debug=False) influx2_write_api.close() influx2_write_api = influx2_client.write_api( write_options=WriteOptions(batch_size=200, flush_interval=120_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2))
def __init__(self): self.client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=False) self.write_api = self.client.write_api( write_options=WriteOptions(write_type=WriteType.batching, batch_size=50_000, flush_interval=10_000))
def init_connection(): client = InfluxDBClient(url="http://localhost:8086", token=token, org=org) write_client = client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2))
def setUp(self) -> None: super().setUp() self.influxDb_client = InfluxDBClient(url="http://localhost:8086", token="my-token", debug=False) self.write_options = WriteOptions(batch_size=10_000, flush_interval=5_000, retry_interval=3_000) self._write_client = WriteApi(influxdb_client=self.influxDb_client, write_options=self.write_options)
def __init__(self, queue): multiprocessing.Process.__init__(self) self.queue = queue self.client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org", debug=False) self.write_api = self.client.write_api( write_options=WriteOptions(write_type=WriteType.batching, batch_size=50_000, flush_interval=10_000))
def start(self): """ Start the submission thread """ self._running = True self._influx = InfluxDBClient(url=self._config['url'], org=self._config['org'], token=self._config['token'], enable_gzip=True) opts = WriteOptions(batch_size=self._batch_size, flush_interval=self._poll_interval * 1000, jitter_interval=5000) self._iwrite = self._influx.write_api(write_options=opts) self._car.register_data(self.data_callback)
def import_data(config, data): points = [] for d in data: point = create_point(d) points.append(point) client = InfluxDBClient(url=config['influxdb_url'], token=config['influxdb_token'], org=config['influxdb_org'], debug=False) write_api = client.write_api(write_options=WriteOptions(batch_size=100, flush_interval=50)) write_api.write(bucket=config['influxdb_bucket'], record=points) write_api.close() print("Imported {} records".format(len(points)))
def _influxdb_write(entry, url, token, org, bucket, debug, verify_ssl, convert): with InfluxDBClient(url=url, token=token, org=org, debug=debug, verify_ssl=verify_ssl) as _client: with _client.write_api(write_options=WriteOptions( batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=2, max_retry_delay=30_000, exponential_base=2)) as _write_client: for entry_id in entry.influxdb_event_buffer: for eventname in entry.influxdb_event_buffer[entry_id]: for k in entry.influxdb_event_buffer[entry_id][eventname]: for d in entry.influxdb_event_buffer[entry_id][ eventname][k]: # Keep only eventdata of really published messages if d['time'] > 0: #params = d['changed_params'] if d['changed_params'] else d['params'] params = d['params'] # Keep only primitive values and convert values params = { x: _influxdb_type_conversion( params[x], x, d['params']) if convert else params[x] for x in params if x != "temporary" and (":" not in x) and isinstance(params[x], (int, str, bool, float)) } # Use keys and "temporary" as tags tags = d['keys'] if 'temporary' in d['params'] and d['params'][ 'temporary']: tags['temporary'] = True if params: _write_client.write( bucket, org, { "measurement": entry_id + "." + eventname, "tags": tags, "fields": params, "time": datetime.datetime.utcfromtimestamp( d['time']), })
def writetodb(data_points): client = InfluxDBClient(url=influx_url, token=influx_token, org=influx_org) try: data_write = client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2)) data_write.write(my_bucket, influx_org, data_points)
def open_con(cls): load_dotenv() cls.token = os.getenv('INFLUXDB_V2_TOKEN') cls.org = os.getenv('INFLUXDB_V2_ORG') cls.bucket = os.getenv('INFLUXDB_V2_BUCKET') cls.url = "http://influxdb:8086" cls.client = InfluxDBClient(url=cls.url, token=cls.token, org=cls.org) print('Connected') cls.write_api = cls.client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2))
def connectInfluxDatabase(self): try: # prepare database self.logger.debug( f'Connecting to Influx with: Host:{self.influx_host}, Port: {self.influx_port}, User: {self.influx_user}, DB: {self.influx_db}' ) if (self.influx_version == 1): pass self.influxClient = DataFrameClient(self.influx_host, self.influx_port, self.influx_user, self.influx_pwd, self.influx_db) elif (self.influx_version == 2): retries = WritesRetry(total=20, backoff_factor=1, exponential_base=1) self.influxClient = InfluxDBClient( url=f"http://{self.influx_host}:{self.influx_port}", token=self.influx_token, org=self.influx_org, retries=retries, timeout=180_000) self.influx_query_api = self.influxClient.query_api() self.influx_write_api = self.influxClient.write_api( write_options=WriteOptions( batch_size=500, write_type=WriteType.synchronous, flush_interval=10_000, jitter_interval=2_000, retry_interval=30_000, max_retries=25, max_retry_delay=60_000, exponential_base=2)) #self.influx_write_api = self.influxClient.write_api(write_options=SYNCHRONOUS) except Exception as e: self.logger.exception('Crash!', exc_info=e) sys.exit(99)
def configure() -> None: """ Retrieve or refresh a configuration from IoT Center. Successful configuration is set as a global IOT_CONFIGURATION dictionary with following properties: * id * influx_url * influx_org * influx_token * influx_bucket * configuration_refresh * default_lon * default_lat * measurement_interval """ global config global config_received global influxdb_client global write_api # Check freshness of configuration if config_received and (datetime.utcnow() - config_received).total_seconds( ) < config['configuration_refresh']: return iot_center_url = os.getenv("IOT_CENTER_URL", "http://localhost:5000") iot_device_id = os.getenv("IOT_DEVICE_ID") # Request to configuration config_fresh = fetch_json(f'{iot_center_url}/api/env/{iot_device_id}') # New or changed configuration if not config and config_fresh != config: config = config_fresh config_received = datetime.utcnow() influxdb_client = InfluxDBClient(url=config['influx_url'], token=config['influx_token'], org=config['influx_org']) write_api = influxdb_client.write_api(write_options=WriteOptions( batch_size=1)) print( f'Received configuration: {json.dumps(config, indent=4, sort_keys=False)}' )
def __influxInit(self): if self.__options.verbosity > 0: print("initializing influx configuration", flush=True) # the only way i could find to see how many points are getting flushed is to remap some internal influx functions # _newHttp will ack points in rabbit WriteApi._origHttp = WriteApi._http WriteApi._http = _newHttp writeOptions = WriteOptions(batch_size=self.__points.maxPoints, flush_interval=self.__points.flushInterval) self.__influxClient = InfluxDBClient(url=self.__options.influx2URL, token=self.__options.influx2Token, org=self.__options.influx2Org, debug=self.__options.debug) self.__influx = self.__influxClient.write_api( write_options=writeOptions) self.__influx.main = self
def run(self): """Initialize ``InfluxDBClient`` and waits for data to writes into InfluxDB.""" # Initialize Client and Write API self.client = InfluxDBClient(**self.kwargs) self.write_api = self.client.write_api( write_options=self.kwargs.get('write_options', WriteOptions()), success_callback=self.kwargs.get('success_callback', _success_callback), error_callback=self.kwargs.get('error_callback', _error_callback), retry_callback=self.kwargs.get('retry_callback', _retry_callback)) # Infinite loop - until poison pill while True: next_record = self.queue_.get() if type(next_record) is _PoisonPill: # Poison pill means break the loop self.terminate() self.queue_.task_done() break self.write_api.write(**next_record) self.queue_.task_done()
def influx_client_write_data_frame(host='http://localhost:8086', df=None, df_time_index_name='', database='', measurement_name='', tag_columns=[]): """ 保存数据 df: 数据 DataFrame df_time_index_name: 数据列(InfluxDB只能使用时间作为主键(唯一标志),这里需要把这列设置为 Index) database: 库名(这里需要现在 InfluxDB 内创建好数据库) measurement_name: 表名 tag_columns: 索引 """ client = InfluxDBClient(url=host, token="", org="", debug=False) write_client = client.write_api( write_options=WriteOptions(batch_size=500, flush_interval=10_000, jitter_interval=2_000, retry_interval=5_000, max_retries=5, max_retry_delay=30_000, exponential_base=2))
elif os.path.exists('config.yaml'): import yaml with open('config.yaml', encoding='utf-8') as config_file: C = None for c in yaml.load_all(config_file, Loader=yaml.SafeLoader): C = c else: raise Exception('No config found') Influx = InfluxDBClient(url=C['influxdb']['url'], org=C['influxdb']['org'], token=C['influxdb']['token'], enable_gzip=True) opts = WriteOptions(batch_size=10000, flush_interval=60000, jitter_interval=5000) write_api = Influx.write_api(write_options=opts) ############################################################################## if __name__ == "__main__": ser = Serial("/dev/ttyUSB0") while True: line = ser.readline() if line[:6] == b'^RSSI:': now = time.time() data = { 'Strength': int(line[6:]), }
# %% cp = forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper', 'measurement']].copy() lines = [ str(cp["measurement"][d]) + ",type=forecast" + " " + "yhat=" + str(cp["yhat"][d]) + "," + "yhat_lower=" + str(cp["yhat_lower"][d]) + "," + "yhat_upper=" + str(cp["yhat_upper"][d]) + " " + str(int(time.mktime(cp['ds'][d].timetuple()))) + "000000000" for d in range(len(cp)) ] # %% from influxdb_client import InfluxDBClient, Point, WriteOptions from influxdb_client.client.write_api import SYNCHRONOUS _write_client = client.write_api( write_options=WriteOptions(batch_size=1000, flush_interval=10000, jitter_interval=2000, retry_interval=5000)) _write_client.write(bucket, org, lines) lines[0:10] # %% [markdown] # To close client: # %% _write_client.__del__() client.__del__()
import pyarrow.parquet as pq from influxdb_client import InfluxDBClient, WriteOptions with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", timeout=0, debug=False) as client: """ You can download NYC TLC Trip Record Data parquet file from https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page """ table = pq.read_table('fhvhv_tripdata_2022-01.parquet') with client.write_api(write_options=WriteOptions( batch_size=50_000)) as write_api: dataframe = table.to_pandas() """ Keep only interesting columns """ keep_df = dataframe[[ 'dispatching_base_num', "PULocationID", "DOLocationID", "pickup_datetime", "dropoff_datetime", "shared_request_flag" ]] print(keep_df.tail().to_string()) write_api.write(bucket="my-bucket", record=keep_df, data_frame_measurement_name="taxi-trip-data", data_frame_tag_columns=[ 'dispatching_base_num', "shared_request_flag"
""" Converts vix-daily.csv into sequence of data point """ data = rx \ .from_iterable(DictReader(open('vix-daily.csv', 'r'))) \ .pipe(ops.map(lambda row: parse_row(row))) client = InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) """ Create client that writes data in batches with 50_000 items. """ write_api = client.write_api( write_options=WriteOptions(batch_size=50_000, flush_interval=10_000)) """ Write data into InfluxDB """ write_api.write(bucket="my-bucket", record=data) write_api.close() """ Querying max value of CBOE Volatility Index """ query = 'from(bucket:"my-bucket")' \ ' |> range(start: 0, stop: now())' \ ' |> filter(fn: (r) => r._measurement == "financial-analysis")' \ ' |> max()' result = client.query_api().query(query=query) """ Processing results
""" Converts vix-daily.csv into sequence of data point """ data = rx \ .from_iterable(DictReader(open('vix-daily.csv', 'r'))) \ .pipe(ops.map(lambda row: parse_row(row))) with InfluxDBClient(url="http://localhost:8086", token="my-token", org="my-org", debug=True) as client: """ Create client that writes data in batches with 50_000 items. """ with client.write_api(write_options=WriteOptions( batch_size=50_000, flush_interval=10_000)) as write_api: """ Write data into InfluxDB """ write_api.write(bucket="my-bucket", record=data) """ Querying max value of CBOE Volatility Index """ query = 'from(bucket:"my-bucket")' \ ' |> range(start: 0, stop: now())' \ ' |> filter(fn: (r) => r._measurement == "financial-analysis")' \ ' |> max()' result = client.query_api().query(query=query) """ Processing results """
import socket return 'iot_sensor,hostname={},type=temperature value={}'.format( socket.gethostname(), temperature) """ Read temperature every minute; distinct_until_changed - produce only if temperature change """ data = rx \ .interval(period=timedelta(seconds=60)) \ .pipe(ops.map(lambda t: sensor_temperature()), ops.distinct_until_changed(), ops.map(lambda temperature: line_protocol(temperature))) _db_client = InfluxDBClient(url="http://localhost:9999", token="my-token", org="my-org", debug=True) """ Create client that writes data into InfluxDB """ _write_api = _db_client.write_api(write_options=WriteOptions(batch_size=1)) _write_api.write(org="my-org", bucket="my-bucket", record=data) """ Call after terminate a script """ atexit.register(on_exit, _db_client, _write_api) input()