def set_query_pattern(cls, queries: Optional[Mapping] = None, **kwargs) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are passed as mappings, where the key is name name of the desired new method representing the query pattern and the value is the actual query pattern. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern dictionary: {"host_load": "SELECT mean(load) FROM cpu_stats WHERE host = '{host}' AND time > now() - {days}d", "peak_load": "SELECT max(load) FROM cpu_stats WHERE host = '{host}' GROUP BY time(1d),host"} :param queries: Mapping (e.g. dictionary) containing query patterns. Can be used in conjunction with kwargs. :param kwargs: Alternative way to pass query patterns. """ if queries is None: queries = {} restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size') for name, query in {**queries, **kwargs}.items(): if any(kw in restricted_kwargs for kw in re.findall('{(\w+)}', query)): warnings.warn(f'Ignoring invalid query pattern: {query}') continue setattr(cls, name, pm(cls.query, query))
def set_query_pattern(cls, name: str, qp: str) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern: ``"SELECT mean(load) FROM cpu_stats WHERE host = '{host}' AND time > now() - {days}d"`` :param name: Name of the query pattern class method. Must be a valid Python identifier. :param qp: Query pattern string """ warnings.warn( "'set_query_pattern' is deprecated and " "will be removed in a future version. " "Define query patterns as functions in your own code instead.", DeprecationWarning, stacklevel=2) restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size', 'parser') if any(kw in restricted_kwargs for kw in re.findall(r'{(\w+)}', qp)): warnings.warn(f'Ignoring invalid query pattern: {qp}') elif not name.isidentifier() or (name in dir(cls) and name not in cls._user_qp): warnings.warn(f'Ignoring invalid query pattern name: {name}') else: cls._user_qp.add(name) setattr(cls, name, pm(cls.query, qp))
def set_query_pattern(cls, queries: Optional[Mapping] = None, **kwargs) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are passed as mappings, with the key being name of the new method and the value the actual query pattern. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern dictionary: .. code:: python {"host_load": "SELECT mean(load) FROM cpu_stats " "WHERE host = '{host}' AND time > now() - {days}d", "peak_load": "SELECT max(load) FROM cpu_stats " "WHERE host = '{host}' GROUP BY time(1d),host"} :param queries: Mapping (e.g. dictionary) containing query patterns. Can be used in conjunction with kwargs. :param kwargs: Alternative way to pass query patterns. """ if queries is None: queries = {} if not isinstance(queries, Mapping): raise ValueError('Query patterns must be passed in a dictionary ' 'or by using keyword arguments') restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size', 'parser') for name, query in {**queries, **kwargs}.items(): if any(kw in restricted_kwargs for kw in re.findall('{(\w+)}', query)): warnings.warn( 'Ignoring invalid query pattern: {}'.format(query)) continue if name in dir(cls) and name not in cls._user_query_patterns: warnings.warn( 'Ignoring invalid query pattern name: {}'.format(name)) continue cls._user_query_patterns.add(name) setattr(cls, name, pm(cls.query, query))
class InfluxDBClient(abc.ABC): def __init__(self, host: str = 'localhost', port: int = 8086, db: str = 'testdb', *, ssl: bool = False, unix_socket: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, database: Optional[str] = None): """ The InfluxDBClient object holds information necessary to interact with InfluxDB. The three main public methods are the three endpoints of the InfluxDB API, namely: 1) InfluxDBClient.ping 2) InfluxDBClient.write 3) InfluxDBClient.query See each of the above methods documentation for further usage details. See also: https://docs.influxdata.com/influxdb/latest/tools/api/ :param host: Hostname to connect to InfluxDB. :param port: Port to connect to InfluxDB. :param mode: Mode in which client should run. Available options are: 'async', 'blocking' and 'dataframe'. - 'async': Default mode. Each query/request to the backend will - 'blocking': Behaves in sync/blocking fashion, similar to the official InfluxDB-Python client. - 'dataframe': Behaves in a sync/blocking fashion, but parsing results into Pandas DataFrames. Similar to InfluxDB-Python's `DataFrameClient`. :param db: Default database to be used by the client. :param ssl: If https should be used. :param unix_socket: Path to the InfluxDB Unix domain socket. :param username: Username to use to connect to InfluxDB. :param password: User password. :param database: Default database to be used by the client. This field is for argument consistency with the official InfluxDB Python client. """ self._url = f'{"https" if ssl else "http"}://{host}:{port}/{{endpoint}}' self.db = database or db @abc.abstractmethod async def _request(self, method: str, url: str, headers: Mapping, body: bytes = b'', stream: bool = False) -> Tuple[int, Mapping, bytes]: """Make an HTTP request.""" async def ping(self) -> dict: """Pings InfluxDB. Returns a dictionary containing the headers of the response from `influxd`. """ status, headers, bytes = await self._request( 'GET', self._url.format(endpoint='ping')) return headers async def write(self, data: Union[PointType, Iterable[PointType]], measurement: Optional[str] = None, tag_columns: Optional[Iterable] = None, **extra_tags) -> bool: """Writes data to InfluxDB. Input can be: 1) a string properly formatted in InfluxDB's line protocol 2) a dictionary-like object containing four keys: 'measurement', 'time', 'tags', 'fields' 3) a Pandas DataFrame with a DatetimeIndex 4) an iterable of one of above Input data in formats 2-4 are parsed to the line protocol before being written to InfluxDB. See also: https://docs.influxdata.com/influxdb/latest/write_protocols/line_protocol_reference/ :param data: Input data (see description above). :param tag_columns: Columns that should be treated as tags (used when writing DataFrames only) :param measurement: Measurement name. Mandatory when when writing DataFrames only. When writing dictionary-like data, this field is treated as the default value for points that do not contain a `measurement` field. :param extra_tags: Additional tags to be added to all points passed. :return: Returns `True` if insert is successful. Raises `ValueError` exception otherwise. """ data = parse_data(data, measurement, tag_columns, **extra_tags) url = self._url.format(endpoint='write') + '?' + urlencode( dict(db=self.db)) status, headers, data = await self._request('POST', url, data=data) if status == 204: return True else: msg = (f'Error writing data ({status}): ' f'{headers.get("X-Influxdb-Error")}') raise InfluxDBError(msg) async def query(self, q: AnyStr, *args, db=None, epoch='ns', chunked=False, chunk_size=None, **kwargs) -> Union[AsyncGenerator, dict]: """Sends a query to InfluxDB. Please refer to the InfluxDB documentation for all the possible queries: https://docs.influxdata.com/influxdb/latest/query_language/ :param q: Raw query string :param args: Positional arguments for query patterns :param db: Database parameter. Defaults to `self.db` :param epoch: Precision level of response timestamps. Valid values: ``{'ns', 'u', 'µ', 'ms', 's', 'm', 'h'}``. :param chunked: If ``True``, makes InfluxDB return results in streamed batches rather than as a single response. Returns an AsyncGenerator which yields responses in the same format as non-chunked queries. :param chunk_size: Max number of points for each chunk. By default, InfluxDB chunks responses by series or by every 10,000 points, whichever occurs first. :param kwargs: Keyword arguments for query patterns :return: Returns an async generator if chunked is ``True``, otherwise returns a dictionary containing the parsed JSON response. """ @async_generator async def _chunked_generator(url, data): status, headers, chunks = await self._request('POST', url, data=data, stream=True) async for chunk in chunks: chunk = json.loads(chunk) check_error(chunk) await yield_(chunk) try: if args: fields = [ i for i in re.findall('{(\w+)}', q) if i not in kwargs ] kwargs.update(dict(zip(fields, args))) db = self.db if db is None else db query = q.format(db=db, **kwargs) except KeyError as e: raise ValueError(f'Missing argument "{e.args[0]}" in {repr(q)}') data = dict(q=query, db=db, chunked=str(chunked).lower(), epoch=epoch) if chunked and chunk_size: data['chunk_size'] = chunk_size url = self._url.format(endpoint='query') if chunked: return _chunked_generator(url, data) status, headers, resp = await self._request('POST', url, data=data) output = json.loads(resp) check_error(output) return output # Built-in query patterns create_database = pm(query, "CREATE DATABASE {db}") drop_database = pm(query, "DROP DATABASE {db}") drop_measurement = pm(query, "DROP MEASUREMENT {measurement}") show_databases = pm(query, "SHOW DATABASES") show_measurements = pm(query, "SHOW MEASUREMENTS") show_retention_policies = pm(query, "SHOW RETENTION POLICIES") show_users = pm(query, "SHOW USERS") select_all = pm(query, "SELECT * FROM {measurement}") show_tag_keys = pm(query, "SHOW TAG KEYS") show_tag_values = pm(query, 'SHOW TAG VALUES WITH key = "{key}"') show_tag_keys_from = pm(query, "SHOW TAG KEYS FROM {measurement}") show_tag_values_from = pm( query, 'SHOW TAG VALUES FROM {measurement} WITH key = "{key}"') @classmethod def set_query_pattern(cls, queries: Optional[Mapping] = None, **kwargs) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are passed as mappings, where the key is name name of the desired new method representing the query pattern and the value is the actual query pattern. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern dictionary: {"host_load": "SELECT mean(load) FROM cpu_stats WHERE host = '{host}' AND time > now() - {days}d", "peak_load": "SELECT max(load) FROM cpu_stats WHERE host = '{host}' GROUP BY time(1d),host"} :param queries: Mapping (e.g. dictionary) containing query patterns. Can be used in conjunction with kwargs. :param kwargs: Alternative way to pass query patterns. """ if queries is None: queries = {} restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size') for name, query in {**queries, **kwargs}.items(): if any(kw in restricted_kwargs for kw in re.findall('{(\w+)}', query)): warnings.warn(f'Ignoring invalid query pattern: {query}') continue setattr(cls, name, pm(cls.query, query))
class InfluxDBClient: def __init__(self, host: str = 'localhost', port: int = 8086, mode: str = 'async', output: str = 'raw', db: Optional[str] = None, *, ssl: bool = False, unix_socket: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, database: Optional[str] = None, loop: Optional[asyncio.BaseEventLoop] = None, ): """ The InfluxDBClient object holds information necessary to interact with InfluxDB. It is async by default, but can also be used as a sync/blocking client. When querying, responses are returned as raw JSON by default, but can also be wrapped in easily iterable wrapper object or be parsed to Pandas DataFrames. The three main public methods are the three endpoints of the InfluxDB API, namely: 1) InfluxDBClient.ping 2) InfluxDBClient.write 3) InfluxDBClient.query See each of the above methods documentation for further usage details. See also: https://docs.influxdata.com/influxdb/latest/tools/api/ :param host: Hostname to connect to InfluxDB. :param port: Port to connect to InfluxDB. :param mode: Mode in which client should run. Available options are: 'async', 'blocking' and 'dataframe'. - 'async': Default mode. Each query/request to the backend will - 'blocking': Behaves in sync/blocking fashion, similar to the official InfluxDB-Python client. :param output: Output format of the response received from InfluxDB. - 'raw': Default format. Returns JSON as received from InfluxDB. - 'iterable': Wraps the raw response in a `InfluxDBResult` or `InfluxDBChunkedResult`, which can be used for easier iteration over retrieved data points. - 'dataframe': Parses results into Pandas DataFrames. Not compatible with chunked responses. :param db: Default database to be used by the client. :param ssl: If https should be used. :param unix_socket: Path to the InfluxDB Unix domain socket. :param username: Username to use to connect to InfluxDB. :param password: User password. :param database: Default database to be used by the client. This field is for argument consistency with the official InfluxDB Python client. :param loop: Asyncio event loop. """ self._loop = asyncio.get_event_loop() if loop is None else loop self._connector = aiohttp.UnixConnector(path=unix_socket, loop=self._loop) if unix_socket else None self._auth = aiohttp.BasicAuth(username, password) if username and password else None self._session = aiohttp.ClientSession(loop=self._loop, auth=self._auth, connector=self._connector) self._url = f'{"https" if ssl else "http"}://{host}:{port}/{{endpoint}}' self.host = host self.port = port self._mode = None self._output = None self._db = None self.tag_cache = defaultdict(lambda: defaultdict(dict)) self.mode = mode self.output = output self.db = database or db @property def mode(self): return self._mode @property def output(self): return self._output @property def db(self): return self._db @mode.setter def mode(self, mode): if mode not in ('async', 'blocking'): raise ValueError('Invalid running mode') self._mode = mode @output.setter def output(self, output): if pd is None and output == 'dataframe': raise ValueError(no_pandas_warning) if output not in ('raw', 'iterable', 'dataframe'): raise ValueError('Invalid output format') self._output = output @db.setter def db(self, db): self._db = db if not db: warnings.warn(f'No default databases is set. ' f'Database must be specified when querying/writing.') elif self.output == 'dataframe' and db not in self.tag_cache: if self.mode == 'async': asyncio.ensure_future(self.get_tag_info(), loop=self._loop) else: self.get_tag_info() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close() def __del__(self): if not self._loop.is_closed() and self._session: asyncio.ensure_future(self._session.close(), loop=self._loop) def __repr__(self): items = [f'{k}={v}' for k, v in vars(self).items() if not k.startswith('_') and k != 'tag_cache'] items.append(f'mode={self.mode}') return f'{type(self).__name__}({", ".join(items)})' @runner async def close(self): if self._session: await self._session.close() self._session = None @runner async def ping(self) -> dict: """Pings InfluxDB. Returns a dictionary containing the headers of the response from `influxd`. """ async with self._session.get(self._url.format(endpoint='ping')) as resp: logger.debug(f'{resp.status}: {resp.reason}') return dict(resp.headers.items()) @runner async def write(self, data: Union[PointType, Iterable[PointType]], options: Optional[dict] = {}, measurement: Optional[str] = None, tag_columns: Optional[Iterable] = None, **extra_tags) -> bool: """Writes data to InfluxDB. Input can be: 1) a string properly formatted in InfluxDB's line protocol 2) a dictionary-like object containing four keys: 'measurement', 'time', 'tags', 'fields' 3) a Pandas DataFrame with a DatetimeIndex 4) an iterable of one of above Input data in formats 2-4 are parsed to the line protocol before being written to InfluxDB. See also: https://docs.influxdata.com/influxdb/latest/write_protocols/line_protocol_reference/ :param data: Input data (see description above). :param options: Request options (db, rp, precision, etc). :param measurement: Measurement name. Mandatory when when writing DataFrames only. When writing dictionary-like data, this field is treated as the default value for points that do not contain a `measurement` field. :param tag_columns: Columns that should be treated as tags (used when writing DataFrames only) :param extra_tags: Additional tags to be added to all points passed. :return: Returns `True` if insert is successful. Raises `ValueError` exception otherwise. """ options = dict({ 'db': self.db }, **options) data = parse_data(data, measurement, tag_columns, **extra_tags) logger.debug(data) url = self._url.format(endpoint='write') + '?' + urlencode(options) async with self._session.post(url, data=data) as resp: if resp.status == 204: return True else: raise InfluxDBWriteError(resp) @runner async def query(self, q: AnyStr, *args, epoch: str = 'ns', chunked: bool = False, chunk_size: Optional[int] = None, db: Optional[str] = None, parser: Optional[Callable] = None, **kwargs) -> ResultType: """Sends a query to InfluxDB. Please refer to the InfluxDB documentation for all the possible queries: https://docs.influxdata.com/influxdb/latest/query_language/ :param q: Raw query string :param args: Positional arguments for query patterns :param db: Database to be queried. Defaults to `self.db`. :param epoch: Precision level of response timestamps. Valid values: ``{'ns', 'u', 'µ', 'ms', 's', 'm', 'h'}``. :param chunked: If ``True``, makes InfluxDB return results in streamed batches rather than as a single response. Returns an AsyncGenerator which yields responses in the same format as non-chunked queries. :param chunk_size: Max number of points for each chunk. By default, InfluxDB chunks responses by series or by every 10,000 points, whichever occurs first. :param kwargs: Keyword arguments for query patterns :param parser: Optional parser function for 'iterable' mode :return: Returns an async generator if chunked is ``True``, otherwise returns a dictionary containing the parsed JSON response. """ async def _chunked_generator(url, data): async with self._session.post(url, data=data) as resp: # Hack to avoid aiohttp raising ValueError('Line is too long') # The number 16 is arbitrary (may be too large/small). resp.content._high_water *= 16 async for chunk in resp.content: chunk = json.loads(chunk) self._check_error(chunk) yield chunk try: if args: fields = [i for i in re.findall('{(\w+)}', q) if i not in kwargs] kwargs.update(dict(zip(fields, args))) db = self.db if db is None else db query = q.format(db=db, **kwargs) except KeyError as e: raise ValueError(f'Missing argument "{e.args[0]}" in {repr(q)}') data = dict(q=query, db=db, chunked=str(chunked).lower(), epoch=epoch) if chunked and chunk_size: data['chunk_size'] = chunk_size url = self._url.format(endpoint='query') if chunked: if self.mode != 'async': raise ValueError("Can't use 'chunked' with non-async mode") g = _chunked_generator(url, data) if self.output == 'raw': return g elif self.output == 'iterable': return InfluxDBChunkedResult(g, parser=parser, query=query) elif self.output == 'dataframe': raise ValueError("Chunked queries are not support with 'dataframe' output") async with self._session.post(url, data=data) as resp: logger.debug(resp) output = await resp.json() logger.debug(output) self._check_error(output) if self.output == 'raw': return output elif self.output == 'iterable': return InfluxDBResult(output, parser=parser, query=query) elif self.output == 'dataframe': return make_df(output, self.tag_cache[self.db]) @staticmethod def _check_error(response): """Checks for JSON error messages and raises Python exception""" if 'error' in response: raise InfluxDBError(response['error']) elif 'results' in response: for statement in response['results']: if 'error' in statement: msg = '{d[error]} (statement {d[statement_id]})' raise InfluxDBError(msg.format(d=statement)) # noinspection PyCallingNonCallable @runner async def get_tag_info(self) -> Optional[dict]: """Gathers tag key/value information for measurements in current database This method sends a series of ``SHOW TAG KEYS`` and ``SHOW TAG VALUES`` queries to InfluxDB and gathers key/value information for all measurements of the active database in a dictionary. This is used internally automatically when using ``dataframe`` mode in order to correctly parse dataframes. """ # noinspection PyCallingNonCallable async def get_measurement_tags(m, cache): keys = (await self.show_tag_keys_from(m))['results'][0] if 'series' not in keys: return for series in keys['series']: cache[series['name']] = defaultdict(list) for tag in chain(*series['values']): tag_values = await self.show_tag_values_from(series['name'], tag) for _, v in tag_values['results'][0]['series'][0]['values']: cache[series['name']][tag].append(v) logger.info(f"Caching tags from all measurements from '{self.db}'") cache = {} state = self.mode, self.output self.mode = 'async' self.output = 'raw' ms = (await self.show_measurements())['results'][0] if 'series' not in ms: self.mode, self.output = state return await asyncio.gather(*[get_measurement_tags(m[0], cache) for m in ms['series'][0]['values']]) for m in cache: cache[m] = {k: v for k, v in cache[m].items()} if cache: self.tag_cache[self._db] = cache self.mode, self.output = state return cache # Built-in query patterns _user_query_patterns = set() create_database = pm(query, "CREATE DATABASE {db}") drop_database = pm(query, "DROP DATABASE {db}") drop_measurement = pm(query, "DROP MEASUREMENT {measurement}") show_databases = pm(query, "SHOW DATABASES") show_measurements = pm(query, "SHOW MEASUREMENTS") show_retention_policies = pm(query, "SHOW RETENTION POLICIES") show_users = pm(query, "SHOW USERS") select_all = pm(query, "SELECT * FROM {measurement}") show_tag_keys = pm(query, "SHOW TAG KEYS") show_tag_values = pm(query, 'SHOW TAG VALUES WITH key = "{key}"') show_tag_keys_from = pm(query, "SHOW TAG KEYS FROM {measurement}") show_tag_values_from = pm(query, 'SHOW TAG VALUES FROM {measurement} WITH key = "{key}"') @classmethod def set_query_pattern(cls, queries: Optional[Mapping] = None, **kwargs) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are passed as mappings, where the key is name name of the desired new method representing the query pattern and the value is the actual query pattern. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern dictionary: {"host_load": "SELECT mean(load) FROM cpu_stats WHERE host = '{host}' AND time > now() - {days}d", "peak_load": "SELECT max(load) FROM cpu_stats WHERE host = '{host}' GROUP BY time(1d),host"} :param queries: Mapping (e.g. dictionary) containing query patterns. Can be used in conjunction with kwargs. :param kwargs: Alternative way to pass query patterns. """ if queries is None: queries = {} if not isinstance(queries, Mapping): raise ValueError('Query patterns must be passed in a dictionary ' 'or by using keyword arguments') restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size', 'parser') for name, query in {**queries, **kwargs}.items(): if any(kw in restricted_kwargs for kw in re.findall('{(\w+)}', query)): warnings.warn(f'Ignoring invalid query pattern: {query}') continue if name in dir(cls) and name not in cls._user_query_patterns: warnings.warn(f'Ignoring invalid query pattern name: {name}') continue cls._user_query_patterns.add(name) setattr(cls, name, pm(cls.query, query))
class InfluxDBClient: def __init__( self, host: str = 'localhost', port: int = 8086, mode: str = 'async', db: str = 'testdb', *, ssl: bool = False, unix_socket: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, database: Optional[str] = None, loop: Optional[asyncio.BaseEventLoop] = None, ): """ The InfluxDBClient object holds information necessary to interact with InfluxDB. It is async by default, but can also be used as a sync/blocking client and even generate Pandas DataFrames from queries. The three main public methods are the three endpoints of the InfluxDB API, namely: 1) InfluxDBClient.ping 2) InfluxDBClient.write 3) InfluxDBClient.query See each of the above methods documentation for further usage details. See also: https://docs.influxdata.com/influxdb/latest/tools/api/ :param host: Hostname to connect to InfluxDB. :param port: Port to connect to InfluxDB. :param mode: Mode in which client should run. Available options are: 'async', 'blocking' and 'dataframe'. - 'async': Default mode. Each query/request to the backend will - 'blocking': Behaves in sync/blocking fashion, similar to the official InfluxDB-Python client. - 'dataframe': Behaves in a sync/blocking fashion, but parsing results into Pandas DataFrames. Similar to InfluxDB-Python's `DataFrameClient`. :param db: Default database to be used by the client. :param ssl: If https should be used. :param unix_socket: Path to the InfluxDB Unix domain socket. :param username: Username to use to connect to InfluxDB. :param password: User password. :param database: Default database to be used by the client. This field is for argument consistency with the official InfluxDB Python client. :param loop: Event loop used for processing HTTP requests. """ self._loop = asyncio.get_event_loop() if loop is None else loop self._connector = aiohttp.UnixConnector( path=unix_socket, loop=self._loop) if unix_socket else None self._auth = aiohttp.BasicAuth( username, password) if username and password else None self._session = aiohttp.ClientSession(loop=self._loop, auth=self._auth, connector=self._connector) self._url = f'{"https" if ssl else "http"}://{host}:{port}/{{endpoint}}' self.host = host self.port = port self.db = database or db self._mode = None self.mode = mode @property def mode(self): return self._mode @mode.setter def mode(self, mode): if mode not in ('async', 'blocking', 'dataframe'): raise ValueError('Invalid mode') self._mode = mode def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close() def __del__(self): if not self._loop.is_closed( ) and self._session and self.mode != 'async': asyncio.ensure_future(self._session.close(), loop=self._loop) def __repr__(self): items = [ f'{k}={v}' for k, v in vars(self).items() if not k.startswith('_') ] items.append(f'mode={self.mode}') return f'{type(self).__name__}({", ".join(items)})' @runner async def close(self): if self._session: await self._session.close() self._session = None @runner async def ping(self) -> dict: """Pings InfluxDB. Returns a dictionary containing the headers of the response from `influxd`. """ async with self._session.get( self._url.format(endpoint='ping')) as resp: logger.debug(f'{resp.status}: {resp.reason}') return dict(resp.headers.items()) @runner async def write(self, data: Union[PointType, Iterable[PointType]], measurement: Optional[str] = None, tag_columns: Optional[Iterable] = None, **extra_tags) -> bool: """Writes data to InfluxDB. Input can be: 1) a string properly formatted in InfluxDB's line protocol 2) a dictionary-like object containing four keys: 'measurement', 'time', 'tags', 'fields' 3) a Pandas DataFrame with a DatetimeIndex 4) an iterable of one of above Input data in formats 2-4 are parsed to the line protocol before being written to InfluxDB. See also: https://docs.influxdata.com/influxdb/latest/write_protocols/line_protocol_reference/ :param data: Input data (see description above). :param tag_columns: Columns that should be treated as tags (used when writing DataFrames only) :param measurement: Measurement name. Mandatory when when writing DataFrames only. When writing dictionary-like data, this field is treated as the default value for points that do not contain a `measurement` field. :param extra_tags: Additional tags to be added to all points passed. :return: Returns `True` if insert is successful. Raises `ValueError` exception otherwise. """ data = parse_data(data, measurement, tag_columns, **extra_tags) logger.debug(data) url = self._url.format(endpoint='write') + '?' + urlencode( dict(db=self.db)) async with self._session.post(url, data=data) as resp: if resp.status == 204: return True else: msg = (f'Error writing data ({resp.status}): ' f'{resp.headers.get("X-Influxdb-Error", resp.reason)}') raise InfluxDBError(msg) @runner async def query(self, q: AnyStr, *args, db=None, epoch='ns', chunked=False, chunk_size=None, **kwargs) -> Union[AsyncGenerator, dict]: """Sends a query to InfluxDB. Please refer to the InfluxDB documentation for all the possible queries: https://docs.influxdata.com/influxdb/latest/query_language/ :param q: Raw query string :param args: Positional arguments for query patterns :param db: Database parameter. Defaults to `self.db` :param epoch: Precision level of response timestamps. Valid values: ``{'ns', 'u', 'µ', 'ms', 's', 'm', 'h'}``. :param chunked: If ``True``, makes InfluxDB return results in streamed batches rather than as a single response. Returns an AsyncGenerator which yields responses in the same format as non-chunked queries. :param chunk_size: Max number of points for each chunk. By default, InfluxDB chunks responses by series or by every 10,000 points, whichever occurs first. :param kwargs: Keyword arguments for query patterns :return: Returns an async generator if chunked is ``True``, otherwise returns a dictionary containing the parsed JSON response. """ async def _chunked_generator(url, data): async with self._session.post(url, data=data) as resp: # Hack to avoid aiohttp raising ValueError('Line is too long') # The number 16 is arbitrary (may be too large/small). resp.content._high_water *= 16 async for chunk in resp.content: chunk = json.loads(chunk) self._check_error(chunk) yield chunk try: if args: fields = [ i for i in re.findall('{(\w+)}', q) if i not in kwargs ] kwargs.update(dict(zip(fields, args))) db = self.db if db is None else db query = q.format(db=db, **kwargs) except KeyError as e: raise ValueError(f'Missing argument "{e.args[0]}" in {repr(q)}') data = dict(q=query, db=db, chunked=str(chunked).lower(), epoch=epoch) if chunked and chunk_size: data['chunk_size'] = chunk_size url = self._url.format(endpoint='query') if chunked: return _chunked_generator(url, data) async with self._session.post(url, data=data) as resp: logger.debug(resp) output = await resp.json() logger.debug(output) self._check_error(output) return output @staticmethod def _check_error(response): """Checks for JSON error messages and raises Python exception""" if 'error' in response: raise InfluxDBError(response['error']) elif 'results' in response: for statement in response['results']: if 'error' in statement: msg = '{d[error]} (statement {d[statement_id]})' raise InfluxDBError(msg.format(d=statement)) # Built-in query patterns create_database = pm(query, "CREATE DATABASE {db}") drop_database = pm(query, "DROP DATABASE {db}") drop_measurement = pm(query, "DROP MEASUREMENT {measurement}") show_databases = pm(query, "SHOW DATABASES") show_measurements = pm(query, "SHOW MEASUREMENTS") show_retention_policies = pm(query, "SHOW RETENTION POLICIES") show_users = pm(query, "SHOW USERS") select_all = pm(query, "SELECT * FROM {measurement}") show_tag_keys = pm(query, "SHOW TAG KEYS") show_tag_values = pm(query, 'SHOW TAG VALUES WITH key = "{key}"') show_tag_keys_from = pm(query, "SHOW TAG KEYS FROM {measurement}") show_tag_values_from = pm( query, 'SHOW TAG VALUES FROM {measurement} WITH key = "{key}"') @classmethod def set_query_pattern(cls, queries: Optional[Mapping] = None, **kwargs) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are passed as mappings, where the key is name name of the desired new method representing the query pattern and the value is the actual query pattern. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern dictionary: {"host_load": "SELECT mean(load) FROM cpu_stats WHERE host = '{host}' AND time > now() - {days}d", "peak_load": "SELECT max(load) FROM cpu_stats WHERE host = '{host}' GROUP BY time(1d),host"} :param queries: Mapping (e.g. dictionary) containing query patterns. Can be used in conjunction with kwargs. :param kwargs: Alternative way to pass query patterns. """ if queries is None: queries = {} restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size') for name, query in {**queries, **kwargs}.items(): if any(kw in restricted_kwargs for kw in re.findall('{(\w+)}', query)): warnings.warn(f'Ignoring invalid query pattern: {query}') continue setattr(cls, name, pm(cls.query, query))
class InfluxDBClient: def __init__( self, host: str = 'localhost', port: int = 8086, mode: str = 'async', output: str = 'json', db: Optional[str] = None, *, ssl: bool = False, unix_socket: Optional[str] = None, username: Optional[str] = None, password: Optional[str] = None, database: Optional[str] = None, loop: Optional[asyncio.AbstractEventLoop] = None, ): """ :class:`~aioinflux.client.InfluxDBClient` holds information necessary to interact with InfluxDB. It is async by default, but can also be used as a sync/blocking client. When querying, responses are returned as parsed JSON by default, but can also be wrapped in easily iterable wrapper object or be parsed to Pandas DataFrames. The three main public methods are the three endpoints of the InfluxDB API, namely: 1. :meth:`~aioinflux.client.InfluxDBClient.ping` 2. :meth:`~aioinflux.client.InfluxDBClient.write` 3. :meth:`~aioinflux.client.InfluxDBClient.query` See each of the above methods documentation for further usage details. See also: https://docs.influxdata.com/influxdb/latest/tools/api/ :param host: Hostname to connect to InfluxDB. :param port: Port to connect to InfluxDB. :param mode: Mode in which client should run. Available options: - ``async``: Default mode. Each query/request to the backend will - ``blocking``: Behaves in sync/blocking fashion, similar to the official InfluxDB-Python client. :param output: Output format of the response received from InfluxDB. - ``json``: Default format. Returns parsed JSON as received from InfluxDB. - ``bytes``: Returns raw, non-parsed JSON binary blob as received from InfluxDB. No error checking is performed. Useful for response caching. - ``dataframe``: Parses results into :py:class`pandas.DataFrame`. Not compatible with chunked responses. :param db: Default database to be used by the client. :param ssl: If https should be used. :param unix_socket: Path to the InfluxDB Unix domain socket. :param username: Username to use to connect to InfluxDB. :param password: User password. :param database: Default database to be used by the client. This field is for argument consistency with the official InfluxDB Python client. :param loop: Asyncio event loop. """ self._loop = asyncio.get_event_loop() if loop is None else loop with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) self._session = aiohttp.ClientSession( loop=self._loop, auth=aiohttp.BasicAuth(username, password) if username and password else None, connector=aiohttp.UnixConnector(path=unix_socket, loop=self._loop) if unix_socket else None, ) self._url = f'{"https" if ssl else "http"}://{host}:{port}/{{endpoint}}' self.host = host self.port = port self.mode = mode self.output = output self.db = database or db @property def mode(self): return self._mode @property def output(self): return self._output @property def db(self): return self._db @mode.setter def mode(self, mode): if mode not in ('async', 'blocking'): raise ValueError('Invalid running mode') self._mode = mode @output.setter def output(self, output): if pd is None and output == 'dataframe': raise ValueError(no_pandas_warning) if output not in ('json', 'bytes', 'iterable', 'dataframe'): raise ValueError('Invalid output format') self._output = output @db.setter def db(self, db): self._db = db if not db: warnings.warn(f'No default databases is set. ' f'Database must be specified when querying/writing.') def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close() def __del__(self): if not self._loop.is_closed() and self._session: asyncio.ensure_future(self._session.close(), loop=self._loop) def __repr__(self): items = [ f'{k}={v}' for k, v in vars(self).items() if not k.startswith('_') ] items.append(f'mode={self.mode}') return f'{type(self).__name__}({", ".join(items)})' @runner async def close(self): if self._session: await self._session.close() self._session = None @runner async def ping(self) -> dict: """Pings InfluxDB. Returns a dictionary containing the headers of the response from ``influxd``. """ async with self._session.get( self._url.format(endpoint='ping')) as resp: logger.debug(f'{resp.status}: {resp.reason}') return dict(resp.headers.items()) @runner async def write( self, data: Union[PointType, Iterable[PointType]], measurement: Optional[str] = None, db: Optional[str] = None, precision: Optional[str] = None, rp: Optional[str] = None, tag_columns: Optional[Iterable] = None, **extra_tags, ) -> bool: """Writes data to InfluxDB. Input can be: 1. A mapping (e.g. ``dict``) containing the keys: ``measurement``, ``time``, ``tags``, ``fields`` 2. A Pandas :class:`~pandas.DataFrame` with a :class:`~pandas.DatetimeIndex` 3. A user defined class decorated w/ :func:`~aioinflux.serialization.usertype.lineprotocol` 4. A string (``str`` or ``bytes``) properly formatted in InfluxDB's line protocol 5. An iterable of one of the above Input data in formats 1-3 are parsed to the line protocol before being written to InfluxDB. See the `InfluxDB docs <https://docs.influxdata.com/influxdb/latest/ write_protocols/line_protocol_reference/>`_ for more details. :param data: Input data (see description above). :param measurement: Measurement name. Mandatory when when writing DataFrames only. When writing dictionary-like data, this field is treated as the default value for points that do not contain a `measurement` field. :param db: Database to be written to. Defaults to `self.db`. :param precision: Sets the precision for the supplied Unix time values. Ignored if input timestamp data is of non-integer type. Valid values: ``{'ns', 'u', 'µ', 'ms', 's', 'm', 'h'}`` :param rp: Sets the target retention policy for the write. If unspecified, data is written to the default retention policy. :param tag_columns: Columns to be treated as tags (used when writing DataFrames only) :param extra_tags: Additional tags to be added to all points passed. :return: Returns ``True`` if insert is successful. Raises :py:class:`ValueError` otherwise. """ if precision is not None: # FIXME: Implement. Related issue: aioinflux/pull/13 raise NotImplementedError( "'precision' parameter is not supported yet") data = serialization.serialize(data, measurement, tag_columns, **extra_tags) logger.debug(data) params = {'db': db or self.db} if rp: params['rp'] = rp url = self._url.format(endpoint='write') async with self._session.post(url, params=params, data=data) as resp: if resp.status == 204: return True raise InfluxDBWriteError(resp) @runner async def query( self, q: AnyStr, *args, epoch: str = 'ns', chunked: bool = False, chunk_size: Optional[int] = None, db: Optional[str] = None, **kwargs, ) -> ResultType: """Sends a query to InfluxDB. Please refer to the InfluxDB documentation for all the possible queries: https://docs.influxdata.com/influxdb/latest/query_language/ :param q: Raw query string :param args: Positional arguments for query patterns :param db: Database to be queried. Defaults to `self.db`. :param epoch: Precision level of response timestamps. Valid values: ``{'ns', 'u', 'µ', 'ms', 's', 'm', 'h'}``. :param chunked: If ``True``, makes InfluxDB return results in streamed batches rather than as a single response. Returns an AsyncGenerator which yields responses in the same format as non-chunked queries. :param chunk_size: Max number of points for each chunk. By default, InfluxDB chunks responses by series or by every 10,000 points, whichever occurs first. :param kwargs: Keyword arguments for query patterns :param parser: Optional parser function for 'iterable' mode :return: Response in the format specified by the combination of ``InfluxDBClient.output`` and ``chunked`` """ # noinspection PyShadowingNames async def _chunked_generator(url, data): async with self._session.post(url, data=data) as resp: # Hack to avoid aiohttp raising ValueError('Line is too long') # The number 16 is arbitrary (may be too large/small). resp.content._high_water *= 16 async for chunk in resp.content: if self.output == 'bytes': yield chunk continue chunk = json.loads(chunk) self._check_error(chunk) yield chunk try: if args: fields = [ i for i in re.findall(r'{(\w+)}', q) if i not in kwargs ] kwargs.update(dict(zip(fields, args))) db = self.db if db is None else db query = q.format(db=db, **kwargs) except KeyError as e: raise ValueError(f'Missing argument "{e.args[0]}" in {repr(q)}') # InfluxDB documentation is wrong regarding `/query` parameters # See https://github.com/influxdata/docs.influxdata.com/issues/1807 if not isinstance(chunked, bool): raise ValueError("'chunked' must be a boolean") data = dict(q=query, db=db, chunked=str(chunked).lower(), epoch=epoch) if chunked and chunk_size: data['chunk_size'] = chunk_size url = self._url.format(endpoint='query') if chunked: if self.mode != 'async': raise ValueError("Can't use 'chunked' with non-async mode") g = _chunked_generator(url, data) if self.output in ('bytes', 'json'): return g elif self.output == 'dataframe': raise ValueError( "Chunked queries are not support with 'dataframe' output") async with self._session.post(url, data=data) as resp: logger.debug(resp) output = await resp.read() logger.debug(output) if self.output == 'bytes': return output output = json.loads(output.decode()) self._check_error(output) if self.output == 'json': return output elif self.output == 'dataframe': return serialization.dataframe.parse(output) @staticmethod def _check_error(response): """Checks for JSON error messages and raises Python exception""" if 'error' in response: raise InfluxDBError(response['error']) elif 'results' in response: for statement in response['results']: if 'error' in statement: msg = '{d[error]} (statement {d[statement_id]})' raise InfluxDBError(msg.format(d=statement)) # Built-in query patterns _user_qp = set() create_database = pm(query, 'CREATE DATABASE "{db}"') drop_database = pm(query, 'DROP DATABASE "{db}"') drop_measurement = pm(query, 'DROP MEASUREMENT "{measurement}"') select_all = pm(query, 'SELECT * FROM "{measurement}"') show_databases = pm(query, "SHOW DATABASES") show_continuous_queries = pm(query, "SHOW CONTINUOUS QUERIES") show_measurements = pm(query, "SHOW MEASUREMENTS") show_retention_policies = pm(query, "SHOW RETENTION POLICIES") show_users = pm(query, "SHOW USERS") show_series = pm(query, 'SHOW SERIES') show_series_from = pm(query, 'SHOW SERIES FROM "{measurement}"') show_tag_keys = pm(query, "SHOW TAG KEYS") show_tag_values = pm(query, 'SHOW TAG VALUES WITH key = "{key}"') show_tag_keys_from = pm(query, 'SHOW TAG KEYS FROM "{measurement}"') show_tag_values_from = pm( query, 'SHOW TAG VALUES FROM "{measurement}" WITH key = "{key}"') @classmethod def set_query_pattern(cls, name: str, qp: str) -> None: """Defines custom methods to provide quick access to commonly used query patterns. Query patterns are plain strings, with optional the named placed holders. Named placed holders are processed as keyword arguments in ``str.format``. Positional arguments are also supported. Sample query pattern: ``"SELECT mean(load) FROM cpu_stats WHERE host = '{host}' AND time > now() - {days}d"`` :param name: Name of the query pattern class method. Must be a valid Python identifier. :param qp: Query pattern string """ warnings.warn( "'set_query_pattern' is deprecated and " "will be removed in a future version. " "Define query patterns as functions in your own code instead.", DeprecationWarning, stacklevel=2) restricted_kwargs = ('q', 'epoch', 'chunked' 'chunk_size', 'parser') if any(kw in restricted_kwargs for kw in re.findall(r'{(\w+)}', qp)): warnings.warn(f'Ignoring invalid query pattern: {qp}') elif not name.isidentifier() or (name in dir(cls) and name not in cls._user_qp): warnings.warn(f'Ignoring invalid query pattern name: {name}') else: cls._user_qp.add(name) setattr(cls, name, pm(cls.query, qp))