def __init__(self, consumer, deserialize): super().__init__(consumer) if self.enabled: self.deserialize = deserialize self._reset_stats() self.influxdb_client = InfluxDBClient(host=INFLUXDB_HOST, db=INFLUXDB_DB, loop=self.loop)
async def main(): client = InfluxDBClient(**influxdb_client_configuration) async for result in scan(5): measurement = RuuviTagMeasurementTest1( name=result.name, temperature=result.temperature, humidity=result.humidity, pressure=result.pressure, voltage=result.voltage, ) asyncio.create_task(client.write(measurement))
async def initialize_database(app): db_host = os.getenv("INFLUXDB_HOST", "localhost") db_port = os.getenv("INFLUXDB_PORT", 8086) db_database = os.getenv("ENVIRONMENT", "aquarium") db = InfluxDBClient(host=db_host, port=db_port, db=db_database) app["influx-db"] = db await db.create_database(db=db_database)
def df_client(): if utils.pd is None: return with InfluxDBClient(db='df_client_test', mode='blocking', output='dataframe') as client: client.create_database() yield client client.drop_database()
async def _init_nosql_dbs(app: web.Application) -> web.Application: """ Wrapper over initializing dbase. :param app: an instance of aiohttp application. :return: an instance of aiohttp application with updates. """ ### # Create connection to databases NoSQLDatabases = type( 'NoSQLDatabases', (SimpleNamespace,), {'mongo': None, 'influx': None} ) app.dbs = NoSQLDatabases() cfg = deep_get(app.shared.extras, 'dbs.mongo', {}) app.dbs.mongo = MongosDBClient(**cfg, loop=app.loop) cfg = deep_get(app.shared.extras, 'dbs.influx', {}) app.dbs.influx = InfluxDBClient(**cfg, loop=app.loop) aws: Tuple[ Union[Awaitable[_Returns_co], Callable[..., _Returns_co]], ...] = ( app.dbs.mongo.ping(), app.dbs.influx.ping(), ) pings = await asyncio.gather(*aws, loop=app.loop, return_exceptions=True) ### # Attention! I apologize in advance, but the code that was in this place I had # to remove that it would be completely not to violate NDA. # ¯\_(ツ)_/¯ # Thanks for understanding! return app
async def configure_connection(settings: Settings): influx.client = InfluxDBClient(host=settings.influx_db_host, port=settings.influx_db_port, database=settings.influx_db_database, username=settings.influx_db_user, password=settings.influx_db_password) logger.info("Connection set up for InfluxDB")
async def main(config: Dict[str, Any]) -> None: """ Main worker """ async with InfluxDBClient(host=config['INFLUXDB_HOST'], port=int(config['INFLUXDB_PORT']), username=config['INFLUXDB_USERNAME'], password=config['INFLUXDB_PASSWORD'], db=config['INFLUXDB_DB']) as client: icmp_redis_pool = await create_redis_pool( (config['REDIS_HOST'], int(config['REDIS_PORT'])), db=int(config['REDIS_DB'])) snmp_redis_pool = await create_redis_pool( (config['REDIS_HOST'], int(config['REDIS_PORT'])), db=int(config['REDIS_DB'])) icmp_queue = AioRedisQueue(redis=icmp_redis_pool, queue_name=config['ICMP_QUEUE_NAME']) snmp_queue = AioRedisQueue(redis=snmp_redis_pool, queue_name=config['SNMP_QUEUE_NAME']) tasks = [] for _ in range(int(config['TASKS'])): icmp_task = asyncio.create_task( influx_callback(icmp_worker(icmp_queue), client)) tasks.append(icmp_task) for _ in range(int(config['TASKS'])): snmp_task = asyncio.create_task( influx_callback(snmp_worker(snmp_queue), client)) tasks.append(snmp_task) await asyncio.gather(*tasks)
def get_influx(influx_config): assert influx_config return InfluxDBClient( host=influx_config['host'], port=influx_config['port'], username=influx_config['username'], password=influx_config['password'], db=influx_config['db'])
async def iter_client(): async with InfluxDBClient(db='iter_client_test', mode='async', output='iterable') as client: await client.create_database() await client.write([p for p in utils.cpu_load_generator(100)]) await yield_(client) await client.drop_database()
async def log(data: dict): try: async with InfluxDBClient(host=InfluxDBParams.STATS_HOST, db=InfluxDBParams.STATS_DB, username=InfluxDBParams.STATS_USER, password=InfluxDBParams.STATS_PASS, timeout=INFLUX_WRITE_TIMEOUT_SEC) as client: await client.write(data) except InfluxDBWriteError as ex: logging.error(f"InfluxDB write error: {str(ex)}")
async def main_coroutine(self): """Main coroutine.""" client = InfluxDBClient(host=self.db_server, db=self.db_name) try: await client.create_database(host=self.db_server, db=self.db_name) except aiohttp.client_exceptions.ClientConnectorError as e: log.error(e) return log_flag = True while self.run_flag: try: cur_key = self.c_params["l_rtt_key"] l_rtt = self.containers[cur_key]["rtt"] for key, attrs in self.containers.items(): query = f'select mean("value") from rtt where ("host" = \'{key}\') and time > now() - 3s fill(0) limit 1' query_res = await client.query(query) series = query_res["results"][0].get("series") if series: values = series[0].get("values") point = float(values[0][-1]) self.containers[key]["rtt"] = point # if current path is down, steer away if point == 0.0: if log_flag and key == cur_key: log.info( "Current path is down! Steering away.") log_flag = False self.containers[key]["rtt"] = self.c_params[ "max_rtt"] await asyncio.sleep(self.frequency) # optimize log.debug(f"current_lowest {self.containers[cur_key]['rtt']}") # find lowest first for key, attrs in self.containers.items(): # if the latency is lower, update lowest rtt key if attrs["rtt"] * 1.20 < l_rtt and attrs["rtt"] > 0: cur_key = key l_rtt = attrs["rtt"] if self.c_params["l_rtt_key"] != cur_key: log_flag = True evc_path = self.containers[cur_key]["evc_path"] log.info(f"changing to lane #{evc_path}") self.c_params["l_rtt_key"] = cur_key async with aiohttp.ClientSession() as session: my_str = f"http://{self.http_server}:{self.http_port}/{self.endpoint}/{evc_path}" print(my_str) data = await self.http_post(session, my_str) log.info(data) except aiohttp.client_exceptions.ClientConnectorError as e: log.error(f"HTTP server {self.http_server} connection refused") return
def __init__(self, db="weather", host="localhost", port=8086, measurement='weather', verbose=False): self.db = db self.measurement = measurement self.client = InfluxDBClient(db=db, host=host, port=port) self.verbose = verbose
async def cache_client(): opts = dict(db='cache_client_test', redis_opts=dict( address='redis://localhost:6379/8', timeout=5, ), cache_expiry=600) async with InfluxDBClient(**opts, mode='async') as client: assert await client.create_database() yield client await client.drop_database() await client._redis.flushdb()
async def push_metric(measurement, tags: Dict, fields: Dict): try: point = {'measurement': measurement, 'tags': tags, 'fields': fields} async with InfluxDBClient(host=INFLUX_HOST, port=INFLUX_PORT, username=INFLUX_USERNAME, password=INFLUX_PASSWORD, db=INFLUX_DB, mode='async') as client: await client.write(point) except Exception as e: print(e) pass
async def get_and_insert_datum_into_influx(verbose=False): point = { 'measurement': 'airsensors', 'tags': { 'host': 'airsensors', 'location': 'home' }, 'fields': await get_aggregate_sensor_datum() } async with InfluxDBClient(host=HOST, db=DB) as client: await client.create_database(db=DB) await client.write(point) if verbose: print(json.dumps(point['fields'], indent=2))
async def efd_client(): df = pd.read_hdf(PATH / 'efd_test.hdf') async with InfluxDBClient(db='client_test', mode='async', output='dataframe') as client: await client.create_database() await client.write(df, measurement='lsst.sal.fooSubSys.test') efd_client = EfdClient('test_efd', db_name='client_test', client=client) # Monkey patch the client to point to an existing schema registry # Note this is only available if on the NCSA VPN efd_client.schema_registry = 'https://lsst-schema-registry-efd.ncsa.illinois.edu' yield efd_client await client.drop_database()
async def fetchMeasurements(loop, display): async with InfluxDBClient(db='testdb', output='iterable') as client: resp = await client.query('show measurements', chunked=True, chunk_size=128) async for chunk in resp.iterchunks(): for result in chunk['results']: series = result['series'] for s in series: for measurement in s['values']: display.AddMeasurement(measurement[0]) try: loop.create_task(display.Refresh()) except Exception as e: print(e)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.launchtime = datetime.datetime.utcnow() # COMMON ATTRIBUTES self.config: dict = json.load(open('config.json', 'r')) self.configs = {} self.overrides: dict = json.load(open('overrides.json', 'r')) self.override_save.start() self.tips = json.load(open('tips.json', 'r')) self.premiumGuilds = [] self.db: asyncpg.pool.Pool = None self.realtime_members = True self.dev = kwargs.pop('dev', False) # CRAB self.crab = '🦀' # LOGGING logging.basicConfig(filename='bot.log', level=logging.INFO) self.logger = logging.getLogger('Fire') stdout = logging.StreamHandler(sys.stdout) stdout.setLevel(logging.INFO) COLOR_FORMAT = colorformat.formatter_message( "[$BOLD%(name)s$RESET][%(levelname)s] %(message)s $RESET($BOLD%(filename)s$RESET:%(lineno)d)" ) stdout.setFormatter(colorformat.ColoredFormatter(COLOR_FORMAT)) self.logger.addHandler(stdout) # SENTRY if 'sentry' in self.config: sentry_sdk.init(self.config['sentry']) # INFLUX if 'influx_user' in self.config and 'influx_pass' in self.config: self.influx = InfluxDBClient(db='firedev' if self.dev else 'fire', username=self.config['influx_user'], password=self.config['influx_pass']) # MODULES self.loadModules() # COMMANDS self.loadCommands() self.cmdresp = {} # EVENTS self.loadEvents()
async def log(user_id: int, event: EventCommand): data = { "measurement": "bot_commands", "time": datetime.now(), "fields": {"event": 1}, "tags": { "user": str(user_id), "command": event.value } } try: async with InfluxDBClient(host=DBParams.STATS_HOST, db=DBParams.STATS_DB, username=DBParams.STATS_USER, password=DBParams.STATS_PASS) as client: await client.write(data) except InfluxDBWriteError as ex: logging.error(f"InfluxDB write error: {str(ex)}")
async def run_test(number_of_day, total_number, type_request): try: end_time = datetime.datetime( 2019, 1, 1, 0, 0, 0) + datetime.timedelta(days=number_of_day) start_time = end_time - datetime.timedelta(days=1) query = QUERY_TYPE_LIST[type_request].replace( 'START_TIME', '{}000000000'.format( int(time.mktime(start_time.timetuple())))).replace( 'END_TIME', '{}000000000'.format( int(time.mktime(end_time.timetuple())))) async with InfluxDBClient(db='benchmarkdb') as client: try: with open( '../data/influx/influx_1sec_{}d.dat'.format( number_of_day), 'rt') as f: network_setup = '' if LATENCY_TYPE or PACKETLOSS_TYPE: network_setup = '_{}_{}'.format( LATENCY_TYPE, PACKETLOSS_TYPE) with open( '{}/{}/{}_{}_{}_{}{}.txt'.format( ANALYSIS_DIRECTORY, BENCHMARK_TEST, BENCHMARK_TEST, type_request, total_number, number_of_day, network_setup), 'w') as fw: i = 0 for i in range(864): bulk_data = [] for j in range(100): l = f.readline() bulk_data.append(l) prev_time = time.time() await client.write(bulk_data) bulk_time = time.time() await client.query(query, db='benchmarkdb') curr_time = time.time() fw.write('{}\t{}\t{}\t{}\n'.format( i, curr_time - prev_time, bulk_time - prev_time, bulk_time - curr_time)) except FileNotFoundError: print( 'You need to generate data first. Please use "data_generator.py" with influx data format.' ) except Exception as e: print('Error: {}'.format(e))
async def vmstat(loop, cmd, hostname, restart=True): proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) # procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- -----timestamp----- # r b swpd free buff cache si so bi bo in cs us sy id wa st UTC # 0 0 2048 286872 1318492 14039020 0 0 1 9 2 1 3 1 96 0 0 2019-01-11 23:11:04 async with InfluxDBClient(db='testdb', output='iterable') as client: done = False while not done: line = await proc.stdout.readline() logging.debug(line) if not line: done = True continue l = line.strip().decode('ascii').split(' ') l = list(filter(None, l)) if len(l) == 19: freeMem = float(l[3]) us = float(l[12]) dayStr = l[17] timeStr = l[18] points = [] points.append({ 'time': '%sT%sZ' % (dayStr, timeStr), 'measurement': 'cpu_load_short', 'tags': {'host': hostname }, 'fields': {'value': us} }) points.append({ 'time': '%sT%sZ' % (dayStr, timeStr), 'measurement': 'free_mem', 'tags': {'host': hostname }, 'fields': {'value': freeMem} }) for point in points: logging.debug(point) await client.write(point) logging.info("%s: Return Code %s", cmd, proc.returncode) if restart: loop.create_task(vmstat(loop, cmd, hostname, restart))
def __init__( self, name: str, https_info: HTTPServerInfo, dbs_info: DBServerInfo, frequency: float = 0.001, timeout: int = 1, ) -> None: """Constructor of Client.""" self.name = name self.h_info = https_info self.d_info = dbs_info self.influx_client = InfluxDBClient( host=dbs_info.addr, db=dbs_info.name, port=dbs_info.port ) self.timeout = timeout self.frequency = frequency
async def _generate_connections( self, loop: Type[asyncio.BaseEventLoop] ) -> Iterator[Type[InfluxDBClient]]: """Iterator that keeps yielding new (connected) clients. It will only yield an influx client if it could successfully ping the remote. There is no limit to total number of clients yielded. """ while True: try: async with InfluxDBClient(host=INFLUX_HOST, db=self._database, loop=loop) as client: await client.ping() LOGGER.info(f'Connected {self}') yield client except ClientConnectionError as ex: # Sleep and try reconnect await asyncio.sleep(RECONNECT_INTERVAL_S)
async def detect(id_user: int, command: Command): data = { "measurement": "bot_commands", "time": datetime.now(), "fields": { "event": 1 }, "tags": { "user": str(id_user), "command": command.value } } try: async with InfluxDBClient(host=EvnDB.STATS_HOST, db=EvnDB.STATS_DB, username=EvnDB.STATS_USER, password=EvnDB.STATS_PASS) as client: await client.write(data) except InfluxDBWriteError as e: logging.error(f"InfluxDB write error: {e}")
async def main(): async with InfluxDBClient(db='testdb', output='iterable') as client: #await client.create_database(db='testdb') for x in range(1,10): current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') point = { 'time': current_time, 'measurement': 'cpu_load_short', 'tags': {'host': 'server01', 'region': 'us-west'}, 'fields': {'value': 0.64+x} } await client.write(point) await asyncio.sleep(.1) resp = await client.query('SELECT value FROM cpu_load_short', chunked=True, chunk_size=128) async for chunk in resp.iterchunks(): yield chunk
async def main(loop): async with InfluxDBClient(host=INFLUX_HOST, db=INFLUX_DB) as client: await client.create_database(db=INFLUX_DB) connection = await aio_pika.connect_robust("amqp://{}:{}@{}/".format( USER, PASSWORD, HOST), loop=loop) queue_name = "metrics" async with connection: channel = await connection.channel() queue = await channel.declare_queue(queue_name, auto_delete=False, durable=True, passive=True) async with queue.iterator() as queue_iter: async for message in queue_iter: async with message.process(): e = Metrics() await e.send(message.body)
async def update_loop(self): await self.bot.wait_until_ready() async with InfluxDBClient(db='xenon') as client: await client.create_database(db='xenon') while True: await asyncio.sleep(60) await self.calculate_total_values() try: for shard_id, stats in self.stats.items(): await client.write([{ "measurement": measurement, "tags": { "shard": str(shard_id) }, "fields": { "value": value } } for measurement, value in stats.items()]) self.build_scheme() except: traceback.print_exc()
async def perfTask(loop, cmd, pid, hostname, restart=True): proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) async with InfluxDBClient(db='testdb', output='iterable') as client: done = False while not done: try: line = await proc.stdout.readline() if not line: done = True continue l = line.strip().decode('ascii').split('#') l = list(filter(None, l)) logging.debug(l) current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') value = int (l[1]) if value: point = { 'time': current_time, 'measurement': 'perf_events', 'tags': { 'type': l[2], 'host': hostname, 'pid': pid }, 'fields': {'value': int(l[1])} } logging.debug(point) await client.write(point) except Exception as e: logging.error(e) done = True logging.info("%s: Return Code %s", cmd, proc.returncode) if restart: loop.create_task(vmstat(loop, cmd, pid, hostname, restart))
async def periodic(timeout): """ Tasks runs every periods setting by the global Variable "tasks_runs_every_n_seconds" """ global session, api_failure_count, api_count, token_start_time, influxdb_client looping_no = 0 token_start_time = time.time() # if using influx client, run following if block if influxdb_write_enable: influxdb_client = InfluxDBClient(host=influxdb_client_host_ip, db=dnac["name"]) await influxdb_client.create_database(db=dnac["name"]) while True: looping_no += 1 logconsole.info(f"looping {str(looping_no)} start") task_run_start_time = time.time() await get_token_refresh() await api_task(session, get_current_time()) elapsed = time.time() - task_run_start_time logconsole.info( f"looping no.{str(looping_no)} took: {elapsed:0.2f}s, api failed/total: {api_failure_count}/{api_count}") # If you need runs infinitely, will NOT break the while loop if not runs_infinitely: if looping_no >= 3: break if timeout > elapsed: await asyncio.sleep(timeout - elapsed) if session: await session.close() return
async def run(self): """Overrides RepeaterFeature.run()""" write_interval = self.app['config']['write_interval'] try: async with InfluxDBClient(host=INFLUX_HOST, db=self._database) as client: await client.ping() LOGGER.info(f'Connected {self}') await client.create_database(db=self._database) while True: await asyncio.sleep(write_interval) if not self._pending: # Do a quick check whether the connection is still alive await client.ping() continue points = self._pending.copy() await client.write(points) LOGGER.debug(f'Pushed {len(points)} points to database') # Make sure to keep points that were inserted during the write self._pending = self._pending[len(points):] except ClientConnectionError as ex: if self._last_ok: LOGGER.warn(f'Database connection failed {self} {ex}') self._last_ok = False await asyncio.sleep(RECONNECT_INTERVAL_S) self._avoid_overflow() except asyncio.CancelledError: raise except Exception: await asyncio.sleep(RECONNECT_INTERVAL_S) raise