# Slow as crap, but worth holding on to for project comparison.
import pandas as pd
from influxdb import InfluxDBClient
bldg = input('bldg ID: ').upper()
source = input("'hotIN', 'coldIN', or 'hotRETURN': ")
print('locating file...\n')
path = "/Users/joseph/Desktop/GRA/InfluxSemesterProject/LLC_BLDG_" + bldg + "/"
file = source + "_LLC_BLDG_" + bldg + "_OCT-4-NOV-13_Testdata.csv"
print('Connecting to InfluxDB...\n')
client = InfluxDBClient(host='influxdbubuntu.bluezone.usu.edu', port=8086)
client.switch_database('LLC_FlowData')
#path="/Users/joseph/Desktop/GRA/InfluxSemesterProject/"
#file = "test.csv"
print('Reading CSV file...\n')
csvReader = pd.read_csv(path + file, sep=',')
print(csvReader.shape)
print(csvReader.columns)
data = len(csvReader)
pbar = ProgressBar(widgets=widgets, maxval=data).start()
def last_data(unique_id, measure_type, measurement_id, period):
"""Return the most recent time and value from influxdb"""
if not str_is_float(period):
return '', 204
if measure_type in ['input', 'math', 'output', 'pid']:
dbcon = InfluxDBClient(
INFLUXDB_HOST,
INFLUXDB_PORT,
INFLUXDB_USER,
INFLUXDB_PASSWORD,
INFLUXDB_DATABASE)
if measure_type in ['input', 'math', 'output', 'pid']:
measure = DeviceMeasurements.query.filter(
DeviceMeasurements.unique_id == measurement_id).first()
else:
return '', 204
if measure:
conversion = Conversion.query.filter(
Conversion.unique_id == measure.conversion_id).first()
else:
conversion = None
channel, unit, measurement = return_measurement_info(
measure, conversion)
if hasattr(measure, 'measurement_type') and measure.measurement_type == 'setpoint':
setpoint_pid = PID.query.filter(PID.unique_id == measure.device_id).first()
if setpoint_pid and ',' in setpoint_pid.measurement:
pid_measurement = setpoint_pid.measurement.split(',')[1]
setpoint_measurement = DeviceMeasurements.query.filter(
DeviceMeasurements.unique_id == pid_measurement).first()
if setpoint_measurement:
conversion = Conversion.query.filter(
Conversion.unique_id == setpoint_measurement.conversion_id).first()
_, unit, measurement = return_measurement_info(setpoint_measurement, conversion)
try:
if period != '0':
query_str = query_string(
unit, unique_id,
measure=measurement, channel=channel,
value='LAST', past_sec=period)
else:
query_str = query_string(
unit, unique_id,
measure=measurement, channel=channel,
value='LAST')
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
number = len(raw_data['series'][0]['values'])
time_raw = raw_data['series'][0]['values'][number - 1][0]
value = raw_data['series'][0]['values'][number - 1][1]
value = float(value)
# Convert date-time to epoch (potential bottleneck for data)
dt = date_parse(time_raw)
timestamp = calendar.timegm(dt.timetuple()) * 1000
live_data = '[{},{}]'.format(timestamp, value)
return Response(live_data, mimetype='text/json')
except KeyError:
logger.debug("No Data returned form influxdb")
return '', 204
except IndexError:
logger.debug("No Data returned form influxdb")
return '', 204
except Exception as e:
logger.exception("URL for 'last_data' raised and error: "
"{err}".format(err=e))
return '', 204
def export_data(unique_id, measurement_id, start_seconds, end_seconds):
"""
Return data from start_seconds to end_seconds from influxdb.
Used for exporting data.
"""
dbcon = InfluxDBClient(
INFLUXDB_HOST,
INFLUXDB_PORT,
INFLUXDB_USER,
INFLUXDB_PASSWORD,
INFLUXDB_DATABASE, timeout=100)
output = Output.query.filter(Output.unique_id == unique_id).first()
input_dev = Input.query.filter(Input.unique_id == unique_id).first()
math = Math.query.filter(Math.unique_id == unique_id).first()
if output:
name = output.name
elif input_dev:
name = input_dev.name
elif math:
name = math.name
else:
name = None
device_measurement = DeviceMeasurements.query.filter(
DeviceMeasurements.unique_id == measurement_id).first()
if device_measurement:
conversion = Conversion.query.filter(
Conversion.unique_id == device_measurement.conversion_id).first()
else:
conversion = None
channel, unit, measurement = return_measurement_info(
device_measurement, conversion)
utc_offset_timedelta = datetime.datetime.utcnow() - datetime.datetime.now()
start = datetime.datetime.fromtimestamp(float(start_seconds))
start += utc_offset_timedelta
start_str = start.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
end = datetime.datetime.fromtimestamp(float(end_seconds))
end += utc_offset_timedelta
end_str = end.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
query_str = query_string(
unit, unique_id,
measure=measurement, channel=channel,
start_str=start_str, end_str=end_str)
if query_str == 1:
flash('Invalid query string', 'error')
return redirect(url_for('routes_page.page_export'))
raw_data = dbcon.query(query_str).raw
if not raw_data or 'series' not in raw_data or not raw_data['series']:
flash('No measurements to export in this time period', 'error')
return redirect(url_for('routes_page.page_export'))
# Generate column names
col_1 = 'timestamp (UTC)'
col_2 = '{name} {meas} ({id})'.format(
name=name, meas=measurement, id=unique_id)
csv_filename = '{id}_{name}_{meas}.csv'.format(
id=unique_id, name=name, meas=measurement)
from flask import Response
import csv
from io import StringIO
def iter_csv(data):
""" Stream CSV file to user for download """
line = StringIO()
writer = csv.writer(line)
writer.writerow([col_1, col_2])
for csv_line in data:
writer.writerow([
str(csv_line[0][:-4]).replace('T', ' '),
csv_line[1]
])
line.seek(0)
yield line.read()
line.truncate(0)
line.seek(0)
response = Response(iter_csv(raw_data['series'][0]['values']), mimetype='text/csv')
response.headers['Content-Disposition'] = 'attachment; filename="{}"'.format(csv_filename)
return response
"Area": "North America",
"Location": "New York City",
"ClientIP": "192.168.0.256"
},
"fields": {
"SessionDuration": 1.2
}
}, {
"measurement": "UserLogins",
"tags": {
"Area": "South America-popo",
"Location": "Lima",
"ClientIP": "192.168.1.256"
},
"fields": {
"SessionDuration": 2.0
}
}]
dbClient = InfluxDBClient('qwerty.com.ar', 8086, 'mim_tp1',
'mim_tp1_transporte', 'AccessHistory')
# Write the time series data points into database - user login details
dbClient.create_database('AccessHistory')
dbClient.write_points(loginEvents)
# Query the IPs from logins have been made
loginRecords = dbClient.query('select * from UserLogins;')
# Print the time series query results
print(loginRecords)
def get_unified_metric(start, end, interval, value, compress): # noqa: E501
"""get_unified_metric
Get **unified metrics** based on speficied start time, end time, time interval and value type. The **start** and **end** time should follow date-time Notation as defined by [RFC 3339, section 5.6](https://tools.ietf.org/html/rfc3339#section-5.6), e.g. `2020-02-12T14:00:00Z`; the time **interval** should follow **duration literals**, which specify a length of time; the **value** type should only be `min`, `max`, `mean`, or `median`. A duration literal is an integer literal followed immediately (with no spaces) by a duration unit, the units include `s`(second), `m`(minute), `h`(hour), `d`(day), `w`(week). # noqa: E501
:param start: start time of time range of the monitoring metrics
:type start: str
:param end: end time of time range of the monitoring metrics
:type end: str
:param interval: time interval for aggregating the monitoring metrics
:type interval: str
:param value: value type of the monitoring metrics
:type value: str
:param compress: return compressed data
:type compress: bool
:rtype: UnifiedMetrics
"""
# Initialization
config = parse_conf()
node_list = parse_host()
host = config["influxdb"]["host"]
port = config["influxdb"]["port"]
start = util.deserialize_datetime(start)
end = util.deserialize_datetime(end)
# When we changed the database, April 28, 2020 11:40:00 AM GMT-05:00 DST
switch_time = 1588092000
start_epoch = int(start.timestamp())
end_epoch = int(end.timestamp())
# Check Sanity
if start_epoch >= switch_time:
dbname = config["influxdb"]["db_monster"]
elif end_epoch <= switch_time:
dbname = config["influxdb"]["database"]
else:
return ErrorMessage(
error_code='400 INVALID_PARAMETERS',
error_message='Due to we switched database on April 28, 2020 \
11:40:00 AM GMT-05:00 DST, currently we do not support \
requesting data with time range falls on this time point.')
if start > end:
return ErrorMessage(
error_code='400 INVALID_PARAMETERS',
error_message='Start time should no larger than end time')
else:
# Initialize influxdb client
client = InfluxDBClient(host=host, port=port, database=dbname)
cpu_count = multiprocessing.cpu_count()
results = []
node_data = {}
job_data = {}
all_jobs_list = []
# Time string used in query_data
st_str = start.strftime('%Y-%m-%dT%H:%M:%SZ')
et_str = end.strftime('%Y-%m-%dT%H:%M:%SZ')
unified_metrics = UnifiedMetrics()
# query_start = time.time()
# Get time stamp
time_list = gen_timestamp(start, end, interval)
epoch_time_list = gen_epoch_timestamp(start, end, interval)
# unified_metrics.time_stamp = epoch_time_list
if compress:
unified_metrics.time_stamp = json_zip(epoch_time_list)
else:
unified_metrics.time_stamp = epoch_time_list
# Get all nodes detail
query_process_data_args = zip(node_list, repeat(client),
repeat(st_str), repeat(et_str),
repeat(interval), repeat(value),
repeat(time_list))
with multiprocessing.Pool(processes=cpu_count) as pool:
results = pool.starmap(query_process_data, query_process_data_args)
# Attach data to node ip addr
for index, node in enumerate(node_list):
if results[index]:
node_data[node] = {
"memory_usage": results[index]["memory_usage"],
"cpu_usage": results[index]["cpu_usage"],
"power_usage": results[index]["power_usage"],
"fan_speed": results[index]["fan_speed"],
"cpu_inl_temp": results[index]["cpu_inl_temp"],
"job_id": results[index]["job_list"]
}
if results[index]["job_set"]:
all_jobs_list.extend(results[index]["job_set"])
if compress:
unified_metrics.nodes_info = json_zip(node_data)
else:
unified_metrics.nodes_info = node_data
# Get all jobs ID
all_jobs_id = list(set(all_jobs_list))
query_job_data_args = zip(repeat(client), all_jobs_id)
# Get all jobs detail
with multiprocessing.Pool(processes=cpu_count) as pool:
results = pool.starmap(query_job_data, query_job_data_args)
for index, job in enumerate(all_jobs_id):
if results[index]:
job_array = False
if "." in results[index]["JobId"]:
job_array = True
if "FinishTime" in results[index]:
finish_time = results[index]["FinishTime"]
else:
finish_time = None
if "NodeList" in results[index]:
node_list = results[index]["NodeList"]
pro_nodelist = process_nodelist(node_list)
else:
node_list = None
job_data[job] = {
"start_time": results[index]["StartTime"],
"submit_time": results[index]["SubmitTime"],
"finish_time": finish_time,
"job_name": results[index]["JobName"],
"user_name": results[index]["User"],
"node_list": pro_nodelist,
"total_nodes": results[index]["TotalNodes"],
"cpu_cores": results[index]["CPUCores"],
"job_array": job_array
}
if compress:
unified_metrics.jobs_info = json_zip(job_data)
else:
unified_metrics.jobs_info = job_data
# total_elapsed = float("{0:.2f}".format(time.time() - query_start))
# # In seconds
# time_range = int(end.timestamp()) - int(start.timestamp())
# with open("requests.log", "a+") as requests_log:
# print(f"{time_range}|{interval}|{value}|{total_elapsed}", \
# file = requests_log)
return unified_metrics
#if power is on, do nothing
else:
print("Power is on.")
#dump current status of power into pickle file
with open('/home/pi/python/data.pickle', 'wb') as f:
pickle.dump(power_out, f, pickle.HIGHEST_PROTOCOL)
#send data to influxdb server
url = ""
port = ""
username = ""
pwd = ""
db_name = ""
client = InfluxDBClient(url, port, username, pwd, db_name)
current_time = str(datetime.datetime.utcnow())
current_status = "On"
if (power_out == 0):
current_status = "Out"
json_body = [
{"measurement": "Power Status",
"time": current_time,
"fields": {
"Status": current_status
}
}
]
client.write_points(json_body)
#!/usr/bin/env python3
import mh_z19
from influxdb import InfluxDBClient
homestats = InfluxDBClient(host='raspberrypi4bserver.local',
database='homestats')
points = [{
'measurement': 'sensor',
'tags': {
'location': 'myRoom'
},
'fields': {
'co2': mh_z19.read()['co2']
}
}]
homestats.write_points(points)
if __name__ == "__main__":
i = 1.1
while (True):
json_body = [{
"measurement": "sensorData",
"tags": {
"source": "bedroom",
"type": "temperature"
},
"fields": {
"value": i
}
}]
i = i * 1.1
if (i >= 100.0):
i = 1.1
client = InfluxDBClient('localhost', 8086, 'admin', 'admin',
'hackaday')
client.create_database('hackaday')
client.write_points(json_body)
result = client.query('select value from sensorData;')
print("Result: {0}".format(result))
time.sleep(1)
def __init__(self, config, **kwargs):
self.config = config
self.client = InfluxDBClient(username=config['indexer']['user'],
password=config['indexer']['password'],
database=config['indexer']['db'],
host=config['indexer'].get('host'))
from influxdb import InfluxDBClient
import time, math, random
#获取当前运行的pid
p1 = psutil.Process(os.getpid())
from influxdb import InfluxDBClient
import time, math, random
while True:
a = psutil.virtual_memory().percent #内存占用率
b = psutil.cpu_percent(interval=1.0) #cpu占用率
json_body = [{
"measurement": "cpu_load_short",
"tags": {
"host": "server01",
"region": "us-west"
},
#"time": "2009-11-10T23:00:00Z",
"fields": {
"cpu": b,
"mem": a
}
}]
client = InfluxDBClient('localhost', 8086, 'root', 'root', 'xxyyxx')
client.create_database('xxyyxx', if_not_exists=False)
client.write_points(json_body)
#result = client.query('select value from cpu_load_short;')
#print("Result: {0}".format(result))
time.sleep(2)
"Luminance": Sensor_Read.Luminance,
"RelativeHumidity": Sensor_Read.RelativeHumidity,
"Temperature": Sensor_Read.Temperature
}
}]
print("Dinesh" + str(json_body))
client_influx.write_points(json_body)
print("Wrote values to DB")
update_problem_pddl()
print("Problem Updated")
pddl_plan = get_pddl_plan()
publish_pddl_plan(pddl_plan)
print(pddl_plan)
client_influx = InfluxDBClient('localhost', 8086, 'root', 'root',
'Sensor_Values')
client_influx.create_database('Sensor_Values')
#MQTT Client instantiation
mqtt_client = mqtt.Client()
# Linking the client with callback implementations
mqtt_client.on_connect = on_connect
mqtt_client.on_message = on_message
# Connecting to the MQTT Broker
mqtt_client.connect(MQTT_BROKER_IP, MQTT_BROKER_PORT, MQTT_KEEPALIVE_INTERVAL)
mqtt_client.loop_start()
mqtt_client.subscribe(MQTT_TOPIC)
print("Subscribed !!")
# snmp constants
HOSTNAME = 'localhost'
COMMUNITY = 'public'
OID_RAM_UNUSED = 'iso.3.6.1.4.1.2021.4.11.0'
OID_DISK_USAGE = 'iso.3.6.1.4.1.2021.9.1.9.1'
OID_CPU_USAGE = 'iso.3.6.1.2.1.25.3.3.1.2' # ...1.2.core_number
URL_DASHBOARD = 'http://localhost:8888'
DB_NAME = 'system'
GRAPHICS_PATH = './Analytics'
# Initializing snmp session
session = Session(hostname=HOSTNAME, community=COMMUNITY, version=2)
# Initializing database
client = InfluxDBClient(HOSTNAME, 8086, 'admin', '1234', DB_NAME)
client.create_database(DB_NAME)
# Open dashboard automatically
web.open_new_tab(URL_DASHBOARD)
print('Running...')
print('Press ctrl+c to stop monitoring and view results.')
while True:
entry = []
cpus = session.walk(OID_CPU_USAGE)
nCores = len(cpus)
load_sum = 0
for c in cpus:
load_sum += int(c.value)
app = Flask(__name__)
#below for flask-sijax
app.config['SIJAX_STATIC_PATH'] = path
app.config['SIJAX_JSON_URI'] = '/static/js/sijax/json2.js'
flask_sijax.Sijax(app)
dict_cnameIDSCnames = {}
with open('cname_id_name1.json') as f:
dict_cnameIDSCnames = json.load(f)
data_cname_sedc = {}
with open('data_cname.json') as f:
data_cname_sedc = json.load(f)
client = InfluxDBClient(host="localhost", port=8086)
client.get_list_database()
client.switch_database("sedc_test")
#get all sedc measurements
measurements = client.query("Show measurements").raw
series = measurements['series']
m_vals = np.array(series[0]['values']).flatten()
ori_response = {}
ori_response["mvals"] = m_vals
ori_response["cnames_all"] = dict_cnameIDSCnames.keys()
def get_TV(data, color):
arr = np.array(data['series'][0]['values'])
arr_vals = arr[:, -1]
def getGardenCallback(self, err, res):
if err:
print err
else:
if configuration["influxdb"]["enabled"]:
influxDbClient = InfluxDBClient(
configuration["influxdb"]["server"],
configuration["influxdb"]["port"],
configuration["influxdb-username"],
configuration["influxdb"]["password"],
configuration["influxdb"]["database"])
try:
influxDbClient.create_database(
configuration["influxdb"]["database"])
except InfluxDBClientError, ex:
print "InfluxDBClientError", ex
influxDbClient.create_retention_policy(
configuration["influxdb"]["policy"],
'INF',
3,
default=True)
for location in res["locations"]:
#print json.dumps(location, indent=2, sort_keys=True)
sensorId = location["sensor"]["sensor_identifier"][-4:].lower()
flower = {}
#flower["sensor_name"] = location["sensor"]["sensor_identifier"]
if location["battery"]["gauge_values"][
"current_value"] is not None:
flower["battery"] = (
"Battery",
int(location["battery"]["gauge_values"]
["current_value"]))
if location["air_temperature"]["gauge_values"][
"current_value"] is not None:
flower["air_temperature"] = (
"Temperature",
float(location["air_temperature"]["gauge_values"]
["current_value"]))
flower["air_temperature_status"] = [
"Temperature Status",
str(location["air_temperature"]
["instruction_key"]).replace(
"air_temperature_", ""),
["good", "too_low", "too_high"]
]
if location["fertilizer"]["gauge_values"][
"current_value"] is not None:
flower["fertilizer"] = (
"Fertilizer",
float(location["fertilizer"]["gauge_values"]
["current_value"]))
flower["fertilizer_status"] = [
"Fertilizer Status",
str(location["fertilizer"]["instruction_key"]).replace(
"fertilizer_", ""),
["good", "too_low", "too_high"]
]
if location["light"]["gauge_values"][
"current_value"] is not None:
flower["light"] = ("Light",
float(location["light"]["gauge_values"]
["current_value"]))
flower["light_status"] = [
"Light Status",
str(location["light"]["instruction_key"]).replace(
"light_", ""), ["good", "too_low", "too_high"]
]
if location["watering"]["soil_moisture"]["gauge_values"][
"current_value"] is not None:
flower["watering"] = (
"Moisture",
float(location["watering"]["soil_moisture"]
["gauge_values"]["current_value"]))
flower["watering_status"] = [
"Moisture Status",
str(location["watering"]["soil_moisture"]
["instruction_key"]).replace("soil_moisture_", ""),
["good", "too_low", "too_high"]
]
lastUtc = ("Updated", str(location["last_sample_utc"]))
if configuration["mqtt"]["enabled"]:
print "Pushing Mqtt", sensorId, ":", configuration["mqtt"][
"prefix"], flower
try:
broadcastMqtt(configuration["mqtt"]["client"],
configuration["mqtt"]["server"],
configuration["mqtt"]["port"],
configuration["mqtt"]["prefix"],
sensorId + "/update",
json.dumps(taflowerg))
except Exception, ex:
print "Error on mqtt broadcast", ex
if configuration["prometheuspush"]["enabled"]:
registry = CollectorRegistry()
for key in flower.keys():
print "Pushing", sensorId, ":", configuration[
"prometheuspush"][
"prefix"] + '_' + key + '_total', "=", flower[
key]
if flower[key][1] is None:
continue
elif type(flower[key][1]) is str:
e = Enum(
configuration["prometheuspush"]["prefix"] +
'_' + key + '_total',
flower[key][0], ['sensorid'],
states=flower[key][2],
registry=registry)
e.labels(sensorid=sensorId).state(flower[key][1])
else:
g = Gauge(
configuration["prometheuspush"]["prefix"] +
'_' + key + '_total',
flower[key][0], ['sensorid'],
registry=registry)
g.labels(sensorid=sensorId).set(flower[key][1])
print "Pushing", sensorId, ":", configuration[
"prometheuspush"][
"prefix"] + '_' + key + '_total', "=", flower[key]
try:
push_to_gateway(
configuration["prometheuspush"]["server"] + ":" +
configuration["prometheuspush"]["port"],
job=configuration["prometheuspush"]["client"] +
"_" + sensorId,
registry=registry)
except:
print "Prometheus not available"
if configuration["influxdb"]["enabled"]:
influxDbJson = [{
"measurement":
configuration["influxdb"]["prefix"],
"tags": {
"sensor": sensorId,
},
"time":
lastUtc[1],
"fields": {}
}]
for key in flower.keys():
influxDbJson[0]["fields"][key] = flower[key][1]
print "Pushing", influxDbJson
try:
influxDbClient.write_points(
influxDbJson,
retention_policy=configuration["influxdb"]
["policy"])
except:
print "Influxdb not available"
#!/usr/bin/python
# Run this script local to retrieve data from serial and write directly into influxDB
import serial
from influxdb import InfluxDBClient
client = InfluxDBClient('192.168.33.56', 8086, 'root', 'root', 'c-base/c-lab')
print(client.query("SHOW DATABASES"))
with serial.Serial('/dev/ttyACM4', 115200, timeout=1) as ser:
while True:
line = ser.readline().strip()
if line:
line = line.split(' ')
if len(line) == 2:
print(line)
tags = line[0].split(',')
data = line[1].split(',')
humidity = data[0].split('=')
temperature = data[1].split('=')
#print(data)
#print(humidity)
#print(temperature)
#print(tags)
sensor = tags[1].split('=')
#print(sensor)
"""
Send packets to magical InfluxDB!
"""
from influxdb import InfluxDBClient
influx_client = InfluxDBClient('localhost', 8086, 'root', 'root', 'sflow')
def mangle_flow(flow):
json_body = [{
"measurement": "realtime",
"tags": flow['metadata'],
"fields": {
"packets":
flow['sample']['sampling_rate'],
"octets":
(flow['frame_length'] - flow['stripped'] - flow['header_length']) *
flow['sample']['sampling_rate'],
}
}]
try:
influx_client.write_points(json_body)
print "Wrote to InfluxDB!"
except:
print "Couldn't write to InfluxDB :("
return flow
influxdb_database = os.getenv("INFLUXDB_DATABASE")
def persists(measurement, fields, time):
logging.info("{} {} {}".format(time, measurement, fields))
influx_client.write_points([{
"measurement": measurement,
"time": time,
"fields": fields
}])
influx_client = InfluxDBClient(
host=influxdb_host,
port=influxdb_port,
database=influxdb_database
)
def get_speed():
logging.info("Calculating speed ...")
s = speedtest.Speedtest()
s.get_best_server()
s.download()
s.upload()
return s.results.dict()
# def loop(sleep):
url = "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull"
r = requests.get(url)
return r.json()['token']
def get_remain(token):
url = "https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest"
header_token = "Bearer " + token
r = requests.head(url, headers={"Authorization": header_token})
return r.headers['RateLimit-Remaining'].split(';')[0]
if __name__ == '__main__':
remain = get_remain(get_token())
client = InfluxDBClient(host=INFLUXDB_HOST,
port=INFLUXDB_PORT,
database=INFLUXDB_DB)
json_body = [{
"measurement":
"dockerhub-ratelimit",
"tags": {
"host": INFLUXDB_TAG,
},
"time":
str(datetime.datetime.utcnow().strftime("%Y-%m-%dT%I:%M:%SZ")),
"fields": {
"remain": remain,
}
}]
client.write_points(json_body)
import time
import random
from influxdb import InfluxDBClient
#Initialize start time of the program
start_time = time.time()
# Configure InfluxDB connection variables
host = "3.86.90.156" # AWS EC2 Instance_ID i-0610264e5f0651062
port = 8086 # default port
user = "******" # the user/password created for the pi, with write access
password = "******"
dbname = "OSNDS" # the database we created earlier
#Initializes the influxDB client
influxClient = InfluxDBClient(host, port, user, password, dbname)
#Initializes a counter variable
counter = 1
accelerationArray = []
#Method to get acceleration data
def getAcceleration():
accelerationArray.append(random.randint(-20, 20))
accelerationArray.append(random.randint(-20, 20))
accelerationArray.append(random.randint(-20, 20))
return accelerationArray
def toInfluxFormat():
# import\ required file -------------------
from influxdb import InfluxDBClient
import pika
import sys
import json
import time
# declarations--------------------
client = InfluxDBClient(host='localhost', port=8086)
#def send_data_to_server(data):
cred=pika.PlainCredentials('admin', 'password')
connection = pika.BlockingConnection(pika.ConnectionParameters(host='34.224.98.65',credentials=cred))
print("connection-",connection)
channel = connection.channel()
channel.queue_declare(queue='local_queue', durable=True)
# code----------------------
while True:
time.sleep(10)
data = read_from_db()
send_data_to_server(data)
def read_from_db():
print("----------------------\nin read from db")
#client.switch_database('ftest')
client.query("alter retention policy onehr on ftest duration 1h replication 1 default")
import zmq
import time
import json
from influxdb import InfluxDBClient, SeriesHelper
USER = ''
PASSWORD = ''
DBNAME = 'carDB'
influx = InfluxDBClient('localhost', 8086, USER, PASSWORD, DBNAME)
influxLineString = ""
latControlstring = 'steerData3,testName=secondRun,active=%s,ff_type=%s ff_type_a=%s,ff_type_r=%s,steer_status=%s,steer_torque_motor=%s,steering_control_active=%s,steer_parameter1=%s,steer_parameter2=%s,steer_parameter3=%s,steer_parameter4=%s,steer_parameter5=%s,steer_parameter6=%s,steer_stock_torque=%s,steer_stock_torque_request=%s,x=%s,y=%s,y1=%s,y2=%s,y3=%s,y3=%s,psi=%s,delta=%s,t=%s,curvature_factor=%s,slip_factor=%s,resonant_period=%s,accel_limit=%s,restricted_steer_rate=%s,ff_angle_factor=%s,ff_rate_factor=%s,pCost=%s,lCost=%s,rCost=%s,hCost=%s,srCost=%s,' + \
'steer_torque_motor=%s,driver_torque=%s,angle_rate_count=%s,angle_rate_desired=%s,avg_angle_rate=%s,future_angle_steers=%s,angle_rate=%s,steer_zero_crossing=%s,' + \
'center_angle=%s,angle_steers=%s,angle_steers_des=%s,angle_offset=%s,self.angle_steers_des_mpc=%s,' + \
'steerRatio=%s,steerKf=%s,steerKpV[0]=%s,steerKiV[0]=%s,steerRateCost=%s,l_prob=%s,r_prob=%s,c_prob=%s,p_prob=%s,' + \
'l_poly[0]=%s,l_poly[1]=%s,l_poly[2]=%s,l_poly[3]=%s,r_poly[0]=%s,r_poly[1]=%s,r_poly[2]=%s,r_poly[3]=%s,p_poly[0]=%s,p_poly[1]=%s,p_poly[2]=%s,p_poly[3]=%s,' + \
'c_poly[0]=%s,c_poly[1]=%s,c_poly[2]=%s,c_poly[3]=%s,d_poly[0]=%s,d_poly[1]=%s,d_poly[2]=%s,lane_width=%s,lane_width_estimate=%s,lane_width_certainty=%s,' + \
'v_ego=%s,p=%s,i=%s,f=%s %s\n'
canFrameString = 'rawCANData,pid=%s,bus=%s First32=%di,Second32=%di,word1=%di,word2=%di,word3=%di,sword4=%di,sword1=%di,sword2=%di,sword3=%di,word4=%di %d\n'
canByteString = 'rawCANBytes,pid=%s,bus=%s,bNum=b%s data=%di %d\n'
#steerFrameString = 'steerData,testName=secondRun lkas_hud_GERNBY1=%s,lkas_hud_GERNBY2=%s,lkas_hud_LKAS_PROBLEM=%s,lkas_hud_LKAS_OFF=%s,lkas_hud_LDW_RIGHT=%s,lkas_hud_BEEP=%s,lkas_hud_LDW_ON=%s,lkas_hud_LDW_OFF=%s,lkas_hud_CLEAN_WINDSHIELD=%s,lkas_hud_DTC=%s,lkas_hud_CAM_TEMP_HIGH=%s,radar_hud_gernby1=%s,radar_hud_gernby2=%s,radar_hud_gernby3=%s,radar_hud_gernby4=%s,radar_hud_gernby5=%s,radar_hud_gernby6=%s,radar_hud_CMBS_OFF=%s,radar_hud_RESUME_INSTRUCTION=%s,stock_steer_request=%s,stock_steer_set_me_x00=%s,stock_steer_set_me_x00_2=%s,stock_steer_steer_torque=%s,lkas_hud_solid_lanes=%s,lkas_hud_steering_required=%s,lkas_hud_set_me_x48=%s,lkas_hud_set_me_x41=%s,lkas_hud_dashed_lanes=%s,speed=%s,lane11=%s,lane12=%s,lane13=%s,lane14=%s,lane15=%s,lane16=%s,lane17=%s,lane18=%s,lane19=%s,lane1A=%s,lane31=%s,lane32=%s,lane33=%s,lane34=%s,lane35=%s,lane36=%s,lane37=%s,lane38=%s,lane39=%s,lane3A=%s,lane51=%s,lane52=%s,lane53=%s,lane54=%s,lane55=%s,lane56=%s,lane57=%s,lane58=%s,lane59=%s,lane5A=%s,lane71=%s,lane72=%s,lane73=%s,lane74=%s,lane75=%s,lane76=%s,lane77=%s,lane78=%s,lane79=%s,lane7A=%s,stock_lane_center=%s,protect_hard=%s,min_steer_limit=%s,OP_STEER_AT_STOCK_LANE_CENTER=%s,lane_diff_1=%s,lane_diff_2=%s,cross_diff=%s,OP_apply_steer=%s,angle_steers=%s,angle_steers_rate=%s,avg_steer_limit=%s,frame=%s,avg_lane_center=%s,sample_count=%s,sent_apply_steer=%s,steer_torque_driver=%s,stock_lane_limit=%s %s\n'
steerFrameString = 'steerData2,testName=secondRun,active=%s ' + \
'lane11=%si,lane12=%si,lane13=%si,lane14=%si,lane15=%si,lane16=%si,lane17=%si,lane18=%si,lane19=%si,lane1A=%si,' + \
'lane31=%si,lane32=%si,lane33=%si,lane34=%si,lane35=%si,lane36=%si,lane37=%si,lane38=%si,lane39=%si,lane3A=%si,' + \
'lane51=%si,lane52=%si,lane53=%si,lane54=%si,lane55=%si,lane56=%si,lane57=%si,lane58=%si,lane59=%si,lane5A=%si,' + \
'lane71=%si,lane72=%si,lane73=%si,lane74=%si,lane75=%si,lane76=%si,lane77=%si,lane78=%si,lane79=%si,lane7A=%si,' + \
'angle_steers=%si,angle_steers_rate=%si,sent_apply_steer=%si,stock_steer_steer_torque=%si,steer_torque_driver=%si,' + \
'stock_lane_center=%si,stock_lane_curvature=%si,avg_lane_center=%si,avg_lane_curvature=%si,avg_steer_angle=%si,avg_steer_error=%si,' + \
'lkas_hud_GERNBY1=%si,lkas_hud_GERNBY2=%si,lkas_hud_LKAS_PROBLEM=%si,lkas_hud_LKAS_OFF=%si,lkas_hud_LDW_RIGHT=%si,lkas_hud_BEEP=%si,' + \
'lkas_hud_LDW_ON=%si,lkas_hud_LDW_OFF=%si,lkas_hud_CLEAN_WINDSHIELD=%si,lkas_hud_DTC=%si,lkas_hud_CAM_TEMP_HIGH=%si,radar_hud_gernby1=%si,' + \
'radar_hud_gernby2=%si,radar_hud_gernby3=%si,radar_hud_gernby4=%si,radar_hud_gernby5=%si,radar_hud_gernby6=%si,radar_hud_CMBS_OFF=%si,' + \
def setUp(self):
self.c = InfluxDBClient(database=INFLUXDB_DATABASE,
username=INFLUXDB_USER,
password=INFLUXDB_PASSWORD)
self.c.delete_series(measurement='Sensor')
from influxdb import InfluxDBClient
'''
Username and password for connect to InfluxDB , Please use Environment Variable.
'''
dbClient = InfluxDBClient('localhost', 8086, 'sabaszx', 'admin', 'trapEvent', ssl=False, verify_ssl=False)
dbClient.switch_database('trapEvent')
#Count number (NEW)
def countSSID(receive, ssid_name):
json_body = [{
"measurement": "countSSID",
"tags": {
"SSIDName": ssid_name,
"type": "known_ssid"},
"fields": {
"item": receive}
}
]
dbClient.write_points(json_body)
'''
#count number
def countAIS(receive):
json_body = [{
"measurement": "countSSID",
"tags": {
"SSIDName": "AIS Smart Login",
"type": "known_ssid"},
"fields": {
import paho.mqtt.client as mqtt
from influxdb import InfluxDBClient
from flask import Flask, request, json
from flask_restful import Resource, Api
import datetime
broker_address = "FILL THIS IN"
client = mqtt.Client()
client.connect(broker_address)
dbclient = InfluxDBClient('0.0.0.0', 8086, 'root', 'root', 'mydb')
app = Flask(__name__)
api = Api(app)
class Test(Resource):
def get(self):
query = 'select mean("value") from "/lightstate" where "time" > now() - 10s'
result = dbclient.query(query)
try:
light_avg = list(
result.get_points(measurement='/lightstate'))[0]['mean']
return {'average': light_avg}
except:
print('exception')
pass
def post(self):
value = request.get_data()
def connect(self):
self.client = InfluxDBClient(host=self.host, port=self.port)
return_msg = daemon.ram_use()
logger.info("[Remote command] Daemon Ram in Use: {msg} MB".format(
msg=return_msg))
elif args.input_force_measurements:
return_msg = daemon.input_force_measurements(
args.input_force_measurements)
logger.info(
"[Remote command] Force acquiring measurements for Input with "
"ID '{id}': Server returned: {msg}".format(
id=args.input_force_measurements, msg=return_msg))
elif args.get_measurement:
client = InfluxDBClient(INFLUXDB_HOST,
INFLUXDB_PORT,
INFLUXDB_USER,
INFLUXDB_PASSWORD,
INFLUXDB_DATABASE,
timeout=5)
query = "SELECT LAST(value) FROM {unit} " \
"WHERE device_id='{id}' " \
"AND channel='{channel}'".format(
unit=args.get_measurement[1],
id=args.get_measurement[0],
channel=args.get_measurement[2])
try:
last_measurement = client.query(query).raw
except requests.exceptions.ConnectionError:
logger.debug("ERROR: Failed to establish a new influxdb "
"connection. Ensure influxdb is running.")
last_measurement = None
def past_data(unique_id, measure_type, measurement_id, past_seconds):
"""Return data from past_seconds until present from influxdb"""
if not str_is_float(past_seconds):
return '', 204
if measure_type == 'tag':
notes_list = []
tag = NoteTags.query.filter(NoteTags.unique_id == unique_id).first()
notes = Notes.query.filter(
Notes.date_time >= (datetime.datetime.utcnow() - datetime.timedelta(seconds=int(past_seconds)))).all()
for each_note in notes:
if tag.unique_id in each_note.tags.split(','):
notes_list.append(
[each_note.date_time.strftime("%Y-%m-%dT%H:%M:%S.000000000Z"), each_note.name, each_note.note])
if notes_list:
return jsonify(notes_list)
else:
return '', 204
elif measure_type in ['input', 'math', 'output', 'pid']:
dbcon = InfluxDBClient(
INFLUXDB_HOST,
INFLUXDB_PORT,
INFLUXDB_USER,
INFLUXDB_PASSWORD,
INFLUXDB_DATABASE)
if measure_type in ['input', 'math', 'output', 'pid']:
measure = DeviceMeasurements.query.filter(
DeviceMeasurements.unique_id == measurement_id).first()
else:
measure = None
if not measure:
return "Could not find measurement"
if measure:
conversion = Conversion.query.filter(
Conversion.unique_id == measure.conversion_id).first()
else:
conversion = None
channel, unit, measurement = return_measurement_info(
measure, conversion)
if hasattr(measure, 'measurement_type') and measure.measurement_type == 'setpoint':
setpoint_pid = PID.query.filter(PID.unique_id == measure.device_id).first()
if setpoint_pid and ',' in setpoint_pid.measurement:
pid_measurement = setpoint_pid.measurement.split(',')[1]
setpoint_measurement = DeviceMeasurements.query.filter(
DeviceMeasurements.unique_id == pid_measurement).first()
if setpoint_measurement:
conversion = Conversion.query.filter(
Conversion.unique_id == setpoint_measurement.conversion_id).first()
_, unit, measurement = return_measurement_info(setpoint_measurement, conversion)
try:
query_str = query_string(
unit, unique_id,
measure=measurement,
channel=channel,
past_sec=past_seconds)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
if 'series' in raw_data and raw_data['series']:
return jsonify(raw_data['series'][0]['values'])
else:
return '', 204
except Exception as e:
logger.debug("URL for 'past_data' raised and error: "
"{err}".format(err=e))
return '', 204
NODE_URL = 'http://nadejde-eos-node.northeurope.cloudapp.azure.com:8080'
BLOCK_PATH = '/v1/chain/get_block'
TRANSACTION_PATH = '/v1/history/get_transaction'
INFO_PATH ='/v1/chain/get_info'
INFLUX_URL = 'http://localhost:8086/write?db=eos'
WRITE_INFLUX = True
WRITE_POSTGRE = True
BATCH_SIZE = 100
#global variables to keep track of prices
#used for buyram actions as there's no way to figure out how much ram was bought without
ramprice = 0.0
ramprice_inc_fee = 0.0
influx_client = InfluxDBClient(database='eos')
pg_config = {
'host': os.environ['PG_EOS_HOST'],
'dbname': os.environ['PG_EOS_DBNAME'],
'user': os.environ['PG_EOS_USER'],
'password': os.environ['PG_EOS_PASSWORD']
}
postgre_connection = psycopg2.connect("host=%(host)s dbname=%(dbname)s user=%(user)s password=%(password)s sslmode=verify-full sslrootcert=/home/nadejde/.postgresql/BaltimoreCyberTrustRoot.crt.pem" % pg_config)
posgre_cursor = postgre_connection.cursor()
#influx
#######
def write_points_influx_line(points):
lines = ""
for point in points:
def async_data(device_id, device_type, measurement_id, start_seconds, end_seconds):
"""
Return data from start_seconds to end_seconds from influxdb.
Used for asynchronous graph display of many points (up to millions).
"""
if device_type == 'tag':
notes_list = []
tag = NoteTags.query.filter(NoteTags.unique_id == device_id).first()
start = datetime.datetime.utcfromtimestamp(float(start_seconds))
if end_seconds == '0':
end = datetime.datetime.utcnow()
else:
end = datetime.datetime.utcfromtimestamp(float(end_seconds))
notes = Notes.query.filter(
and_(Notes.date_time >= start, Notes.date_time <= end)).all()
for each_note in notes:
if tag.unique_id in each_note.tags.split(','):
notes_list.append(
[each_note.date_time.strftime("%Y-%m-%dT%H:%M:%S.000000000Z"), each_note.name, each_note.note])
if notes_list:
return jsonify(notes_list)
else:
return '', 204
dbcon = InfluxDBClient(
INFLUXDB_HOST,
INFLUXDB_PORT,
INFLUXDB_USER,
INFLUXDB_PASSWORD,
INFLUXDB_DATABASE)
if device_type in ['input', 'math', 'output', 'pid']:
measure = DeviceMeasurements.query.filter(
DeviceMeasurements.unique_id == measurement_id).first()
else:
measure = None
if not measure:
return "Could not find measurement"
if measure:
conversion = Conversion.query.filter(
Conversion.unique_id == measure.conversion_id).first()
else:
conversion = None
channel, unit, measurement = return_measurement_info(
measure, conversion)
# Set the time frame to the past year if start/end not specified
if start_seconds == '0' and end_seconds == '0':
# Get how many points there are in the past year
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
value='COUNT')
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
count_points = raw_data['series'][0]['values'][0][1]
# Get the timestamp of the first point in the past year
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
limit=1)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
first_point = raw_data['series'][0]['values'][0][0]
end = datetime.datetime.utcnow()
end_str = end.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
# Set the time frame to the past start epoch to now
elif start_seconds != '0' and end_seconds == '0':
start = datetime.datetime.utcfromtimestamp(float(start_seconds))
start_str = start.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
end = datetime.datetime.utcnow()
end_str = end.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
value='COUNT',
start_str=start_str,
end_str=end_str)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
count_points = raw_data['series'][0]['values'][0][1]
# Get the timestamp of the first point in the past year
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
start_str=start_str,
end_str=end_str,
limit=1)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
first_point = raw_data['series'][0]['values'][0][0]
else:
start = datetime.datetime.utcfromtimestamp(float(start_seconds))
start_str = start.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
end = datetime.datetime.utcfromtimestamp(float(end_seconds))
end_str = end.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
value='COUNT',
start_str=start_str,
end_str=end_str)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
count_points = raw_data['series'][0]['values'][0][1]
# Get the timestamp of the first point in the past year
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
start_str=start_str,
end_str=end_str,
limit=1)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
first_point = raw_data['series'][0]['values'][0][0]
start = datetime.datetime.strptime(
influx_time_str_to_milliseconds(first_point),
'%Y-%m-%dT%H:%M:%S.%f')
start_str = start.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
logger.debug('Count = {}'.format(count_points))
logger.debug('Start = {}'.format(start))
logger.debug('End = {}'.format(end))
# How many seconds between the start and end period
time_difference_seconds = (end - start).total_seconds()
logger.debug('Difference seconds = {}'.format(time_difference_seconds))
# If there are more than 700 points in the time frame, we need to group
# data points into 700 groups with points averaged in each group.
if count_points > 700:
# Average period between input reads
seconds_per_point = time_difference_seconds / count_points
logger.debug('Seconds per point = {}'.format(seconds_per_point))
# How many seconds to group data points in
group_seconds = int(time_difference_seconds / 700)
logger.debug('Group seconds = {}'.format(group_seconds))
try:
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
value='MEAN',
start_str=start_str,
end_str=end_str,
group_sec=group_seconds)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
return jsonify(raw_data['series'][0]['values'])
except Exception as e:
logger.error("URL for 'async_data' raised and error: "
"{err}".format(err=e))
return '', 204
else:
try:
query_str = query_string(
unit, device_id,
measure=measurement,
channel=channel,
start_str=start_str,
end_str=end_str)
if query_str == 1:
return '', 204
raw_data = dbcon.query(query_str).raw
return jsonify(raw_data['series'][0]['values'])
except Exception as e:
logger.error("URL for 'async_data' raised and error: "
"{err}".format(err=e))
return '', 204
import pandas as pd
from influxdb import InfluxDBClient
client = InfluxDBClient(host='floodnet-influxdb.sonycproject.com',
ssl=True,
port=0,
database='db0')
measure = 'distance'
table = 'msg'
app_id = 'deployment_one_app'
dev_id = 'sensor_2'
days_back = 7
query = 'SELECT "%s" ' \
'FROM "%s" ' \
'WHERE "app_id" = \'%s\' ' \
'AND "dev_id" = \'%s\' ' \
'AND "time" >= now() - %id'\
% (measure, table, app_id, dev_id, days_back)
data = client.query(query)
value_data = data.raw['series'][0]['values']
df = pd.DataFrame(value_data, columns=['datetime', 'distance'])
df['datetime'] = pd.to_datetime(df['datetime'], format='%Y-%m-%dT%H:%M:%S.%f')
df['localtime'] = df['datetime'].dt.tz_convert('US/Eastern')
print(df)
