def connect():
"""
Connect to InfluxDB.
"""
try:
print('Connection to InfluxDB...')
client = InfluxDBClient(url=URL, token=TOKEN, org=ORG)
except Exception as e:
print('Could not connect to InfluxDB.', e)
return e
else:
print('Connection to InfluxDB successfully established.')
return client
def __init__(
self,
host: str = None,
port: int = None,
organization: str = "GEWV",
token: str = None,
client: InfluxDBClient = None,
verify_ssl: bool = True,
):
if client is None:
if host is None:
raise Exception(
"Missing Host Address for Timeseries DB Client.")
if port is None:
raise Exception("Missing Port for Timeseries DB Client.")
if token is None:
raise Exception("Missing Token for Timeseries DB Client.")
protocol = "https" if verify_ssl else "http"
self._client = InfluxDBClient(
url=f"{protocol}://{host}:{port}",
token=token,
verify_ssl=verify_ssl,
)
if len(organization) != 16:
# receive id of the org and store the info
self._org_api = self._client.organizations_api()
self._org_id = self.get_org_id_by_name(org_name=organization)
if self._org_id is None:
raise Exception(
f"The organization {organization} dont exists in InfluxDB. Break execution."
)
self._client.org = self._org_id
else:
self._client.org = organization
else:
self._client = client
self._org_api = self._client.organizations_api()
self._write_api = self._client.write_api(write_options=SYNCHRONOUS)
self._query_api = self._client.query_api()
self._bucket_api = self._client.buckets_api()
self._grafana_api = GrafanaApi(host=host, port=3000, use_tls=False)
def __init__(self, config: InfluxDBConfig):
self._config = config
self._buffer = list()
self._log = logging.getLogger(__name__)
self._client = None
if config.enable:
self._log.info('Enabled, creating client')
self._client = InfluxDBClient(
url=config.url,
token=config.token.get_secret_value(),
org=config.org)
P_R.register(self)
def refreshLatencyGraphs(secondsToRun):
startTime = datetime.now()
with open('statsByParentNode.json', 'r') as j:
parentNodes = json.loads(j.read())
with open('statsByDevice.json', 'r') as j:
devices = json.loads(j.read())
print("Retrieving device statistics")
devices = getLatencies(devices, secondsToRun)
print("Computing parent node statistics")
parentNodes = getParentNodeStats(parentNodes, devices)
print("Writing data to InfluxDB")
bucket = influxDBBucket
org = influxDBOrg
token = influxDBtoken
url="http://localhost:8086"
client = InfluxDBClient(
url=url,
token=token,
org=org
)
write_api = client.write_api(write_options=SYNCHRONOUS)
queriesToSend = []
for device in devices:
if device['tcpLatency'] != None:
p = Point('Latency').tag("Device", device['hostname']).tag("ParentNode", device['ParentNode']).tag("Type", "Device").field("TCP Latency", device['tcpLatency'])
queriesToSend.append(p)
for parentNode in parentNodes:
if parentNode['tcpLatency'] != None:
p = Point('Latency').tag("Device", parentNode['parentNodeName']).tag("ParentNode", parentNode['parentNodeName']).tag("Type", "Parent Node").field("TCP Latency", parentNode['tcpLatency'])
queriesToSend.append(p)
write_api.write(bucket=bucket, record=queriesToSend)
print("Added " + str(len(queriesToSend)) + " points to InfluxDB.")
client.close()
#with open('statsByParentNode.json', 'w') as infile:
# json.dump(parentNodes, infile)
#with open('statsByDevice.json', 'w') as infile:
# json.dump(devices, infile)
endTime = datetime.now()
durationSeconds = round((endTime - startTime).total_seconds())
print("Graphs updated within " + str(durationSeconds) + " seconds.")
def main():
# By default the arguments are:
my_token = 'rbuCuV6gRHPJRPIRrLB3kOp874S5mUywVUGXJIUe_o1bf2HpxSqy7E6VB9ZUHKzMK4vGNqo6g6TZipJ2PIEXog=='
my_org = "UNIPI"
bucket = "Bucket"
interface = "en0"
if len(sys.argv) == 5:
my_token = sys.argv[1]
my_org = sys.argv[2]
bucket = sys.argv[3]
interface = sys.argv[4]
else:
if len(sys.argv) != 1:
print("Error: Number of arguments")
exit(1)
# Query for received bytes
query1 = 'from(bucket: "' + bucket + '") |> range(start:-1h, stop: now()) ' \
'|> filter(fn: (r) => r._measurement == "net" )' \
'|> filter(fn: (r) => r._field == "bytes_recv" )' \
'|> filter(fn: (r) => r.interface == "' + interface + '" )' \
'|> derivative(unit: 1s, nonNegative: true,columns: ["_value"])'
# Query for sent bytes
query2 = 'from(bucket: "' + bucket + '") |> range(start:-1h, stop: now()) ' \
'|> filter(fn: (r) => r._measurement == "net" ) ' \
'|> filter(fn: (r) => r._field == "bytes_sent" ) ' \
'|> filter(fn: (r) => r.interface == "' + interface + '" ) ' \
'|> derivative(unit: 1s, nonNegative: true, columns: ["_value"] )'
client = InfluxDBClient(url="http://localhost:9999", token=my_token, org=my_org)
query_api = client.query_api()
data1 = query_api.query_data_frame(query=query1)
data2 = query_api.query_data_frame(query=query2)
try:
plt.plot(data1['_time'], data1['_value'], label='Bytes Received')
plt.plot(data2['_time'], data2['_value'], label='Bytes Sent')
except:
print("Error: check the arguments")
exit(1)
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05),
fancybox=True, shadow=True, ncol=5)
plt.show()
# Close client
client.__del__()
print("close client")
def test_check_write_permission_by_empty_data(self):
client = InfluxDBClient(url="http://localhost:9999",
token="my-token-wrong",
org="my-org")
write_api = client.write_api(write_options=SYNCHRONOUS)
with self.assertRaises(ApiException) as cm:
write_api.write("my-bucket", self.org, b'')
exception = cm.exception
self.assertEqual(401, exception.status)
self.assertEqual("Unauthorized", exception.reason)
client.__del__()
def send(options, influxdata):
client = InfluxDBClient(url=options["server"],
token=options["token"],
org=options["organization"],
timeout=6000)
logger.debug("connection established")
write_api = client.write_api(write_options=ASYNCHRONOUS)
write_api.write(options["bucket"], options["organization"], influxdata)
logger.debug("writing to bucket %s", options["bucket"])
logger.debug("writing to organization %s", options["organization"])
logger.debug("writing data %s", influxdata)
write_api.__del__()
client.__del__()
logger.debug("closing connection")
def load_data_printer():
url = "http://octopi.local/api/printer"
headers = {"X-Api-Key": X_API_KEY, "Content-Type": "application/json"}
res_printer = requests.get(url, headers=headers).json()
print(res_printer["state"]["flags"])
print(res_printer["temperature"]["bed"])
print(res_printer["temperature"]["tool0"])
dbClient = InfluxDBClient(url=INFLUXDB_CLOUD_HOST, token=INFLUXDB_TOKEN)
write_api = dbClient.write_api()
for data_point in data_convert_printer(res_printer):
write_api.write(INFLUXDB_BUCKETID, INFLUXDB_ORGID, data_point)
def __init__(self, ff_config):
self.ic = []
for iconfig in ff_config["influxdb_connections"]:
if is_influx2_db(iconfig):
self.ic.append(
InfluxDBClient(url=iconfig["url"], token=iconfig["token"]))
else:
self.ic.append(
influxdb.InfluxDBClient(host=iconfig["address"],
port=iconfig["port"],
username=iconfig["user"],
password=iconfig["pass"],
database=iconfig["database"]))
self.config = ff_config
def initialize_client() -> Optional[InfluxDBClient]:
influxdb_settings = InfluxdbIntegrationSettings.get_solo()
if not influxdb_settings.enabled:
logger.debug(
'INFLUXDB: Integration disabled in settings (or due to an error previously)'
)
return None
use_secure_connection = influxdb_settings.secure in (
InfluxdbIntegrationSettings.SECURE_CERT_NONE,
InfluxdbIntegrationSettings.SECURE_CERT_REQUIRED,
)
if use_secure_connection:
server_base_url = 'https://{}:{}'.format(influxdb_settings.hostname,
influxdb_settings.port)
else:
server_base_url = 'http://{}:{}'.format(influxdb_settings.hostname,
influxdb_settings.port)
logger.debug('INFLUXDB: Initializing InfluxDB client for "%s"',
server_base_url)
influxdb_client = InfluxDBClient(
url=server_base_url,
token=influxdb_settings.api_token,
verify_ssl=influxdb_settings.secure ==
InfluxdbIntegrationSettings.SECURE_CERT_REQUIRED,
timeout=settings.DSMRREADER_CLIENT_TIMEOUT * 1000, # Ms!
)
# logger.debug('INFLUXDB: InfluxDB client/server status: "%s"', influxdb_client.ready().status)
if influxdb_client.buckets_api().find_bucket_by_name(
influxdb_settings.bucket) is None: # pragma: nocover
logger.debug('INFLUXDB: Creating InfluxDB bucket "%s"',
influxdb_settings.bucket)
try:
influxdb_client.buckets_api().create_bucket(
bucket_name=influxdb_settings.bucket,
org=influxdb_settings.organization)
except Exception as e:
InfluxdbIntegrationSettings.objects.update(enabled=False)
logger.error(
'Failed to instantiate InfluxDB connection, disabling InfluxDB integration'
)
raise e
return influxdb_client
def setUp(self) -> None:
super(TasksApiTest, self).setUp()
self.organization = self.find_my_org()
self.authorization = self.add_tasks_authorization(self.organization)
self.client.close()
self.client = InfluxDBClient(self.host, self.authorization.token, debug=self.conf.debug)
self.tasks_api = self.client.tasks_api()
tasks = self.tasks_api.find_tasks()
for task in tasks:
if task.name.endswith("-IT"):
self.tasks_api.delete_task(task.id)
def __init__(self,
url,
org,
bucket,
token,
size=50,
flush_interval=10_000):
self.logger = getLogger(self.__class__.__name__)
self.org = org
self.bucket = bucket
client = InfluxDBClient(url=url, token=token)
write_options = WriteOptions(batch_size=size,
flush_interval=flush_interval)
self.write_api = client.write_api(write_options=write_options)
def test_query_without_credentials(self):
_client = InfluxDBClient(url="http://localhost:9999", token="my-token-wrong-credentials", org="my-org")
with self.assertRaises(ApiException) as ae:
query = 'from(bucket: "my-bucket")' \
'|> range(start: 2020-02-19T23:30:00Z, stop: now())' \
f'|> filter(fn: (r) => r._measurement == "my-measurement")'
_client.query_api().query_data_frame(query=query)
exception = ae.exception
self.assertEqual(401, exception.status)
self.assertEqual("Unauthorized", exception.reason)
_client.close()
def put(self, data: Dict[str, Any]) -> None:
client = InfluxDBClient(url=self.host, token=self.token)
write_api = client.write_api(write_options=SYNCHRONOUS)
for sensor_name, values in data.items():
point = (
Point("climate")
.tag("sensor", sensor_name)
.field("battery", values["battery"])
.field("temperature", values["temperature"])
.field("humidity", values["humidity"])
.field("pressure", values["pressure"])
.time(datetime.utcnow(), WritePrecision.NS)
)
write_api.write(self.bucket, self.org, point)
def _connect(self):
while not self.write_api:
client = InfluxDBClient(url=self.url,
token=self.auth_token,
org=self.org)
# get the orgID from the name:
try:
organizations_api = client.organizations_api()
orgs = organizations_api.find_organizations()
except:
self.client = None
logging.warning('Error connecting to the InfluxDB API. '
'Please confirm that InfluxDB is running and '
'that the authentication token is correct.'
'Sleeping before trying again.')
time.sleep(5)
continue
# Look up the organization id for our org
our_org = next((org for org in orgs if org.name == self.org), None)
if not our_org:
logging.fatal('Can not find org "%s" in InfluxDB', self.org)
raise RuntimeError('Can not find org "%s" in InfluxDB' %
self.org)
self.org_id = our_org.id
# get the bucketID from the name:
bucket_api = client.buckets_api()
bucket = bucket_api.find_bucket_by_name(self.bucket_name)
# if the bucket does not exist then try to create it
if bucket:
self.bucket_id = bucket.id
else:
try:
logging.info('Creating new bucket for: %s',
self.bucket_name)
new_bucket = bucket_api.create_bucket(
bucket_name=self.bucket_name, org_id=self.org_id)
self.bucket_id = new_bucket.id
except:
logging.fatal('Can not create InfluxDB bucket "%s"',
self.bucket_name)
raise RuntimeError('Can not create InfluxDB bucket "%s"' %
self.bucket_name)
self.write_api = client.write_api(write_options=ASYNCHRONOUS)
def test_proxy(self):
self._start_proxy_server()
self.client.close()
self.client = InfluxDBClient(
url=self.host,
token=self.auth_token,
proxy=f"http://localhost:{self.httpd.server_address[1]}",
proxy_headers={'ProxyHeader': 'Val'})
ready = self.client.ready()
self.assertEqual(ready.status, "ready")
self.assertEqual(1, len(InfluxDBClientTestIT.httpRequest))
self.assertEqual(
'Val',
InfluxDBClientTestIT.httpRequest[0].headers.get('ProxyHeader'))
def writetodb(data_points):
client = InfluxDBClient(url=influx_url,
token=influx_token,
org=influx_org)
try:
data_write = client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
data_write.write(my_bucket, influx_org, data_points)
def write_to_influxdb(video_src,sum):
print(counter)
log.info("writing to db for video src..", video_src)
client = InfluxDBClient(config.influxdb_url, config.influxdb_token)
point = Point("crowd_count_camera")\
.field("current_crowd_count", int(sum))\
.field("video_src", video_src)
# point = Point.field("location", "bangalore").field("cameraId", "cam1").measurement("crowd-count").field("value",5)
write_api = client.write_api(write_options=SYNCHRONOUS)
write_api.write(config.infuxdb_bucket, config.influxdb_org, point)
log.info("crowd_count", sum, "written to db..")
def load_data_job():
url = "http://octopi.local/api/job"
# Data request
headers = {"X-Api-Key": X_API_KEY, "Content-Type": "application/json"}
res_job = requests.get(url, headers=headers).json()
print(res_job)
# Connect InfluxDB Clould
dbClient = InfluxDBClient(url=INFLUXDB_CLOUD_HOST, token=INFLUXDB_TOKEN)
write_api = dbClient.write_api()
# Data Serialize
for data_point in data_convert_job(res_job):
write_api.write(INFLUXDB_BUCKETID, INFLUXDB_ORGID, data_point)
def write_to_influx_cloud(bucket, influxkey, influxurl, influxorg, data):
with InfluxDBClient(url=influxurl,
token=influxkey,
org=influxorg,
verify_ssl=False) as client:
write_api = client.write_api(write_options=SYNCHRONOUS)
# data = "responsetime,url=www.gg.com fcp=0.5,toi=1.2,si=0.5,fmp=0.5"
write_api.write(bucket, influxorg, data)
# query = 'from(bucket: "BlackFriday2021") |> range(start: -1h)'
# tables = client.query_api().query(query, org=influxorg)
# for table in tables:
# for record in table.records:
# print(record)
client.close()
def test(options):
answer = False
try:
client = InfluxDBClient(url=options["server"],
token=options["token"],
org=options["organization"],
timeout=10000)
query_api = client.query_api()
query_api.query_stream('from(bucket:"ups") |> range(start: -10m)')
answer = True
except Exception as e:
logger.error('could not connect to InfluxDB server: %s.', str(e))
finally:
client.__del__()
return answer
def __init__(
self,
url="http://localhost:8086",
token="IpLnoNkWhqmnSLO2ieeqmHejYrrokycO5Be8HRgM6UI1S_CO-Py2_opA2E1z6iCzJrv5U_gHGVHh5JMCFsgwjQ=="
):
# You can generate a Token from the "Tokens Tab" in the UI @ localhost:9999
self.org = "vwa"
self.bucket = "vwa"
self.client = InfluxDBClient(url=url, token=token)
self.write_api = self.client.write_api(write_options=SYNCHRONOUS)
self.b = Batcher(500, 5, self._send)
self.q = self.client.query_api()
def upload(request):
organisation_ = "*****@*****.**"
token_ = "okx-Wk8zR33DYZxZP91T10ZTfUFmMz866DCKfE3Z8l5azgUT3QLIIdzk6rATGgCdAyuaAbWrReSi9KfchjW0kg=="
bucket_ = "test Bucket"
url_ = "https://eu-central-1-1.aws.cloud2.influxdata.com"
client = InfluxDBClient(url=url_, token=token_, org=organisation_)
write_api = client.write_api(write_options=SYNCHRONOUS)
if request.method == 'POST':
uploaded_file = request.FILES['filename']
if uploaded_file.name.endswith('.csv'):
messages.error(request, 'Not a csv file')
csv_data = uploaded_file.read().decode('UTF-8')
io_data = io.StringIO(csv_data)
next(io_data)
status = ["red", "green", "orange"]
tl_type = ["standard", "pedestrian", "turn"]
car_capacity = [20, 25]
for column in csv.reader(io_data, delimiter=',', quotechar="|"):
random_index = random.randint(1)
index = random.randint(2)
num_cars = random.randint(20)
json_body = [{
"measurement": "traffic_lights",
"tags": {
"tl_id": column[0],
"tl_location": "Austin, Texas",
"tl_status": status[index],
"tl_type": tl_type[index],
"tl_name": column[8],
"tl_intersection": column[2],
"tl_operation_state": column[4],
"tl_operation_processed_time": column[5]
},
"fields": {
"car_capacity": car_capacity[random_index],
"num_lanes": 3,
"num_cars": num_cars
},
"time": None
}]
write_api.write(bucket=bucket_,
org=organisation_,
record=json_body)
else:
print("hello")
return render(request, 'main_app/base.html')
def open_con(cls):
load_dotenv()
cls.token = os.getenv('INFLUXDB_V2_TOKEN')
cls.org = os.getenv('INFLUXDB_V2_ORG')
cls.bucket = os.getenv('INFLUXDB_V2_BUCKET')
cls.url = "http://influxdb:8086"
cls.client = InfluxDBClient(url=cls.url, token=cls.token, org=cls.org)
print('Connected')
cls.write_api = cls.client.write_api(
write_options=WriteOptions(batch_size=500,
flush_interval=10_000,
jitter_interval=2_000,
retry_interval=5_000,
max_retries=5,
max_retry_delay=30_000,
exponential_base=2))
def import_data(config, data):
points = []
for d in data:
point = create_point(d)
points.append(point)
client = InfluxDBClient(url=config['influxdb_url'],
token=config['influxdb_token'],
org=config['influxdb_org'],
debug=False)
write_api = client.write_api(
write_options=WriteOptions(batch_size=100, flush_interval=50))
write_api.write(bucket=config['influxdb_bucket'], record=points)
write_api.close()
print("Imported {} records".format(len(points)))
def __open(self):
"""
opens connection to database and sets __client attribute
Raises
------
Exception
throws error if connection fails
"""
try:
client = InfluxDBClient(url=self.__url, token=self.__token)
self.__client = client
except Exception as err:
print("Something went wrong: {}".format(err))
def create_client(config: tp.Dict) -> InfluxDBClient:
'''
Returns an InfluxDBClient initialized with config `url` and `token` params
returned by `get_config`
Inputs:
[tp.Dict]
token [str]: INFLUXDB_TOKEN env representing an InfluxDB token
url [str]: INFLUXDB_URL env representing an InfluxDB url
Outputs:
[InfluxDBClient]: InfluxDB client connection instance
'''
return InfluxDBClient(url=config['url'],
token=config['token'],
debug=False)
def __init__(self,
url: str,
bucket_id: str,
org: str,
token: str,
logger: Logger,
buffer_size: int = 100) -> None:
self.bucket_id = bucket_id
self.org = org
self.values_buffer = []
self.client = InfluxDBClient(url=url,
token=token,
org=org,
enable_gzip=True)
self.write_client = self.client.write_api(write_options=SYNCHRONOUS)
self.buffer_size = buffer_size
self.logger = logger
def setUp(self) -> None:
# https://github.com/gabrielfalcao/HTTPretty/issues/368
import warnings
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*")
warnings.filterwarnings("ignore", category=PendingDeprecationWarning, message="isAlive*")
httpretty.enable()
httpretty.reset()
conf = influxdb_client.configuration.Configuration()
conf.host = "http://localhost"
conf.debug = False
self.influxdb_client = InfluxDBClient(url=conf.host, token="my-token")
write_options = WriteOptions(batch_size=2, flush_interval=5_000, retry_interval=3_000)
self._write_client = WriteApi(influxdb_client=self.influxdb_client, write_options=write_options)
def get_events(measurement):
client = InfluxDBClient(url=host, token=token, org=org)
query_api = client.query_api()
tables = query_api.query(
'from(bucket: "capitol_tracker") |> range(start: 1700-01-01T00:01:00Z) |> filter(fn: (r) => r["_measurement"] == "event")'
)
result = dict()
# {'result': '_result', 'table': 2, '_start': datetime.datetime(1883, 7, 20, 2, 28, 48, 249303, tzinfo=datetime.timezone.utc), '_stop': datetime.datetime(2020, 6, 11, 2, 28, 48, 249303, tzinfo=datetime.timezone.utc), '_time': datetime.datetime(2016, 12, 5, 0, 0, tzinfo=datetime.timezone.utc), '_value': 2.0, '_field': 'lifecycle', '_measurement': 'event_candidate', 'effect': 'Gained_a_degree', 'name': 'Graduation'}
for table in tables:
for record in table.records:
print(record.values)
lifecycle_enum = dict([(1.0, "stop"), (0.0, "start")])
if record.values["_value"] == 2.0:
event = {}
event["name"] = record.values["name"].replace("_", " ")
event["effect"] = record.values["effect"].replace("_", " ")
event["time"] = record.values["_time"]
if "event_id" in record.values:
event["event_id"] = record.values["event_id"].replace(
"_", " ")
result[record.values["name"]] = event
elif record.values["_value"] < 2.0:
event = {}
if record.values["name"] in result:
event = result.get(record.values["name"])
event["name"] = record.values["name"].replace("_", " ")
event["effect"] = record.values["effect"].replace("_", " ")
if not "time" in event:
event["time"] = {}
event["time"][lifecycle_enum.get(
record.values["_value"])] = record.values["_time"]
result[record.values["name"]] = event
return result.values()
