class PortfolioOrderInfluxDBLoaderBolt(Bolt): # TODO use passed in options # This spout may need to throttle while doing bulk inserts def initialize(self, storm_conf, context): #TODO debug storm_conf, context self.client = InfluxDBClient("streamparse-box", 8086, "root", "root", "prices_development") def process(self, tup): print("-----") print(tup.values) order = PortfolioOrder(*tup.values) print(order) # tags are dimensions, fields are facts payload = { "measurement": "portfolio_orders", "tags": { "taxpayer_id": str(order.taxpayer_id), "portfolio_order_id": str(order.portfolio_order_id), "order_type": str(order.order_type) }, "fields": { "amount": order.amount, "priority": order.priority }, "timestamp": order.timestamp } #portfolio = json.loads(tup.values[4]) # add portfolio components to tags #for component in portfolio: # tag_name = "portfolio_component_"+component.get("symbol") # tag_value = component.get("weight") # portfolio_order.get("tags").update({tag_name:tag_value}) #print(payload) self.client.write_points([payload])
def test_query_bad_precision(self): cli = InfluxDBClient() with self.assertRaisesRegexp( Exception, "Invalid time precision is given. \(use 's', 'm', 'ms' or 'u'\)" ): cli.query('select column_one from foo', time_precision='g')
def test_query_chunked(self): cli = InfluxDBClient(database='db') example_object = { 'points': [ [1415206250119, 40001, 667], [1415206244555, 30001, 7], [1415206228241, 20001, 788], [1415206212980, 10001, 555], [1415197271586, 10001, 23] ], 'measurement': 'foo', 'columns': [ 'time', 'sequence_number', 'val' ] } example_response = \ json.dumps(example_object) + json.dumps(example_object) with requests_mock.Mocker() as m: m.register_uri( requests_mock.GET, "http://localhost:8086/db/db/series", text=example_response ) self.assertListEqual( cli.query('select * from foo', chunked=True), [example_object, example_object] )
def test_get_continuous_queries(self): cli = InfluxDBClient(database='db') with requests_mock.Mocker() as m: # Tip: put this in a json linter! example_response = '[ { "name": "continuous queries", "columns"' \ ': [ "time", "id", "query" ], "points": [ [ ' \ '0, 1, "select foo(bar,95) from \\"foo_bar' \ 's\\" group by time(5m) into response_times.' \ 'percentiles.5m.95" ], [ 0, 2, "select perce' \ 'ntile(value,95) from \\"response_times\\" g' \ 'roup by time(5m) into response_times.percen' \ 'tiles.5m.95" ] ] } ]' m.register_uri( requests_mock.GET, "http://localhost:8086/db/db/series", text=example_response ) self.assertListEqual( cli.get_list_continuous_queries(), [ 'select foo(bar,95) from "foo_bars" group ' 'by time(5m) into response_times.percentiles.5m.95', 'select percentile(value,95) from "response_times" group ' 'by time(5m) into response_times.percentiles.5m.95' ] )
def __init__(self, hass, influx_conf, query): """Initialize the sensor.""" from influxdb import InfluxDBClient, exceptions self._name = query.get(CONF_NAME) self._unit_of_measurement = query.get(CONF_UNIT_OF_MEASUREMENT) value_template = query.get(CONF_VALUE_TEMPLATE) if value_template is not None: self._value_template = value_template self._value_template.hass = hass else: self._value_template = None database = query.get(CONF_DB_NAME) self._state = None self._hass = hass where_clause = query.get(CONF_WHERE) where_clause.hass = hass influx = InfluxDBClient( host=influx_conf['host'], port=influx_conf['port'], username=influx_conf['username'], password=influx_conf['password'], database=database, ssl=influx_conf['ssl'], verify_ssl=influx_conf['verify_ssl']) try: influx.query("select * from /.*/ LIMIT 1;") self.connected = True self.data = InfluxSensorData( influx, query.get(CONF_GROUP_FUNCTION), query.get(CONF_FIELD), query.get(CONF_MEASUREMENT_NAME), where_clause) except exceptions.InfluxDBClientError as exc: _LOGGER.error("Database host is not accessible due to '%s', please" " check your entries in the configuration file and" " that the database exists and is READ/WRITE.", exc) self.connected = False
class InfluxUplinkPlugin(AbstractPlugin, AggregateResultListener): '''InfluxDB data uploader''' SECTION = 'influx' @staticmethod def get_key(): return __file__ def __init__(self, core): AbstractPlugin.__init__(self, core) AggregateResultListener.__init__(self) self.client = None self.decoder = None def get_available_options(self): return ["address", "port", "tank_tag", "grafana_root", "grafana_dashboard"] def start_test(self): self.start_time = datetime.datetime.now() def end_test(self, retcode): self.end_time = datetime.datetime.now() + datetime.timedelta(minutes=1) return retcode def configure(self): '''Read configuration''' self.tank_tag = self.get_option("tank_tag", "none") address = self.get_option("address", "localhost") port = int(self.get_option("port", "8086")) self.client = InfluxDBClient( address, port, 'root', 'root', 'mydb') grafana_root = self.get_option("grafana_root", "http://localhost/") grafana_dashboard = self.get_option("grafana_dashboard", "tank-dashboard") uuid=self.core.get_uuid() LOG.info( "Grafana link: {grafana_root}" "dashboard/db/{grafana_dashboard}?var-uuid={uuid}&from=-5m&to=now".format( grafana_root=grafana_root, grafana_dashboard=grafana_dashboard, uuid=uuid, ) ) self.decoder = Decoder(self.tank_tag, uuid) aggregator = self.core.get_plugin_of_type(AggregatorPlugin) aggregator.add_result_listener(self) def aggregate_second(self, data): """ @data: SecondAggregateData """ if self.client: points = self.decoder.decode_aggregate(data) self.client.write_points(points, 's') def monitoring_data(self, data): if self.client: points = self.decoder.decode_monitoring(data) self.client.write_points(points, 's')
def test_write_points_batch(self): dummy_points = [ {"measurement": "cpu_usage", "tags": {"unit": "percent"}, "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.34}}, {"measurement": "network", "tags": {"direction": "in"}, "time": "2009-11-10T23:00:00Z", "fields": {"value": 123.00}}, {"measurement": "network", "tags": {"direction": "out"}, "time": "2009-11-10T23:00:00Z", "fields": {"value": 12.00}} ] expected_last_body = ( "network,direction=out,host=server01,region=us-west " "value=12.0 1257894000000000000\n" ) with requests_mock.Mocker() as m: m.register_uri(requests_mock.POST, "http://localhost:8086/write", status_code=204) cli = InfluxDBClient(database='db') cli.write_points(points=dummy_points, database='db', tags={"host": "server01", "region": "us-west"}, batch_size=2) self.assertEqual(m.call_count, 2) self.assertEqual(expected_last_body, m.last_request.body.decode('utf-8'))
def test_request_retry_raises(self, mock_request): """Test that three requests errors will not be handled.""" class CustomMock(object): """Create custom mock object for test.""" def __init__(self): self.i = 0 def connection_error(self, *args, **kwargs): """Handle a connection error for the CustomMock object.""" self.i += 1 if self.i < 4: raise requests.exceptions.HTTPError else: r = requests.Response() r.status_code = 200 return r mock_request.side_effect = CustomMock().connection_error cli = InfluxDBClient(database='db') with self.assertRaises(requests.exceptions.HTTPError): cli.write_points(self.dummy_points)
def test_random_request_retry_raises(self, mock_request): """Test a random number of conn errors plus one will not be handled.""" class CustomMock(object): """Create custom mock object for test.""" def __init__(self, retries): self.i = 0 self.retries = retries def connection_error(self, *args, **kwargs): """Handle a connection error for the CustomMock object.""" self.i += 1 if self.i < self.retries + 1: raise requests.exceptions.ConnectionError else: r = requests.Response() r.status_code = 200 return r retries = random.randint(1, 5) mock_request.side_effect = CustomMock(retries).connection_error cli = InfluxDBClient(database='db', retries=retries) with self.assertRaises(requests.exceptions.ConnectionError): cli.write_points(self.dummy_points)
def test_write(self): with requests_mock.Mocker() as m: m.register_uri( requests_mock.POST, "http://localhost:8086/write", status_code=204 ) cli = InfluxDBClient(database='db') cli.write( {"database": "mydb", "retentionPolicy": "mypolicy", "points": [{"measurement": "cpu_load_short", "tags": {"host": "server01", "region": "us-west"}, "timestamp": "2009-11-10T23:00:00Z", "fields": {"value": 0.64}}]} ) self.assertEqual( json.loads(m.last_request.body), {"database": "mydb", "retentionPolicy": "mypolicy", "points": [{"measurement": "cpu_load_short", "tags": {"host": "server01", "region": "us-west"}, "timestamp": "2009-11-10T23:00:00Z", "fields": {"value": 0.64}}]} )
def test_request_retry(self, mock_request): """Test that two connection errors will be handled.""" class CustomMock(object): """Create custom mock object for test.""" def __init__(self): self.i = 0 def connection_error(self, *args, **kwargs): """Handle a connection error for the CustomMock object.""" self.i += 1 if self.i < 3: raise requests.exceptions.ConnectionError r = requests.Response() r.status_code = 204 return r mock_request.side_effect = CustomMock().connection_error cli = InfluxDBClient(database='db') cli.write_points( self.dummy_points )
def main(): """ Main demo program. Takes samples of computer metrics and stores them into InfluxDB. """ influxClient = InfluxDBClient( host=HOST, port=PORT, username=USERNAME, password=PASSWORD, ssl=USE_SSL ) setupDatabase(influxClient) for i in xrange(SAMPLE_COUNT): print "Collection sample of your computer metrics..." if i % 10 == 0: print "\t({} samples remaining before I quit)".format(SAMPLE_COUNT - i) # Sample computer for metrics. sample = getSample() # Create an object to write to InfluxDB containing all sample measurements. payload = createInfluxPayload(sample) # Write all measurements. influxClient.write_points(payload) # Wait. time.sleep(SAMPLE_INTERVAL_SECONDS)
def test_write_points_udp(self): cli = InfluxDBClient( 'localhost', self.influxd_inst.http_port, 'root', '', database='db', use_udp=True, udp_port=self.influxd_inst.udp_port ) cli.write_points(dummy_point) # The points are not immediately available after write_points. # This is to be expected because we are using udp (no response !). # So we have to wait some time, time.sleep(3) # 3 sec seems to be a good choice. rsp = self.cli.query('SELECT * FROM cpu_load_short') self.assertEqual( # this is dummy_points : [ {'value': 0.64, 'time': '2009-11-10T23:00:00Z', "host": "server01", "region": "us-west"} ], list(rsp['cpu_load_short']) )
def main(): print 'please enter host ip:' host = raw_input() print 'please enter port:' port = raw_input() print 'please enter user:'******'please enter password:'******'please enter database:' db = raw_input() _db_client = InfluxDBClient(host, port, username=user, password=pwd, database=db, ssl=False, verify_ssl=False, timeout=5) print '------------------\n------------------' print 'please enter date in the format: yyyy-mm-dd:' date = raw_input() print 'please enter utc(! 2 hours before summertime !) time in the format: hh-mm-ss:' time = raw_input() print 'please enter event caption:' caption = raw_input() out = [{'measurement': 'event', 'time' : date + 'T' + time + '.000000Z', 'fields' : {'text' : caption}}] # time MUST be UTC! (2 hours less than summertime) try: print time.ctime() + ': send data...' , print _db_client.write_points(out) except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError, InfluxDBClientError, InfluxDBServerError) as e: print e print time.ctime() + ": Error while sending data..."
def query(): # aggregation every four minutes getCurrDBs() queryDb = raw_input("Enter the database that you're querying from: ") client = InfluxDBClient('localhost', 8086, 'root', 'root', queryDb) minTime, maxTime = getTimeRange(client) # collection of the mininum results in the 4mins windows results = [] windowStart = minTime iterator = 1 while iso8601.parse_date(windowStart) < iso8601.parse_date(maxTime): # upper bound for the 4mins window windwoEnd = iso8601.parse_date(windowStart) + datetime.timedelta(minutes=4) if windwoEnd > iso8601.parse_date(maxTime): windwoEnd = iso8601.parse_date(maxTime) windwoEnd = str(windwoEnd).replace(' ', 'T') print '==============' + ' Window ' + str(iterator) + ' ' + '==============' print "Parsing time window: " + windowStart + ' - ' + str(windwoEnd) countQuerymsg = "SELECT COUNT(TEMP) FROM /.*/ WHERE time >= " + "'" + windowStart + "'" + ' AND time <= ' + "'" + windwoEnd + "'" count = client.query(countQuerymsg) # rule: query MIN only when time stamp in the 4mins window > 1 # get the min based on the rule currMin = queryCases(client, windowStart, windwoEnd, count) results.append(currMin) windowStart = windwoEnd iterator += 1 print results print "Num of time windows parsed: " + str(iterator)
class InfluxIndexer(Indexer): def __init__(self, config, **kwargs): self.config = config self.client = InfluxDBClient( username=config['indexer']['user'], password=config['indexer']['password'], database=config['indexer']['db'], host=config['indexer'].get('host')) def index(self, points): measurements = [] for p in points: measurements.append({ 'measurement': 'policy-metrics', 'time': p['Timestamp'], 'fields': { 'rcount': p['Sum'], 'runit': p['Unit']}, 'tags': { 'region': p['Region'], 'account': p['Account'], 'policy': p['Policy'], 'env': p['Env'], 'division': p['Division'], 'resource': p.get('ResType', ''), 'metric': p['MetricName'], 'namespace': p['Namespace']}}) self.client.write_points(measurements)
def create_influxdb_database(self): db = "ns_" + str(self.id) client = InfluxDBClient(**INFLUXDB['default']) print("Create database: " + db) client.create_database(db) print("Add grants") client.grant_privilege("all", db, self.operator.name)
def insert(host = 'localhost', port = 8086, user = '******', password = '******', db = 'orders'): client = InfluxDBClient(host, port, user, password, db) query = [] for i in Data.orders: query.extend(i.to_json()) #client.delete_series(database = db, measurement = 'orders') client.write_points(query)
def write_influxdb_list(logger, host, port, user, password, dbname, data): """ Write an entry into an Influxdb database example: write_influxdb('localhost', 8086, 'mycodo', 'password123', 'mycodo_db', data_list_of_dictionaries) :return: success (0) or failure (1) :rtype: bool :param host: What influxdb address :type host: str :param port: What influxdb port :type port: int :param user: What user to connect to influxdb with :type user: str :param password: What password to supply for Influxdb user :type password: str :param dbname: What Influxdb database name to write to :type dbname: str :param data_list_of_dictionaries: The data being entered into the Influxdb database. See controller_sensor.py function addMeasurementInfluxdb() :type data_list_of_dictionaries: list of dictionaries """ client = InfluxDBClient(host, port, user, password, dbname) try: client.write_points(data) return 0 except Exception as except_msg: logger.debug('Failed to write measurements to influxdb (Device ID: ' '{}). Data that was submitted for writing: {}. ' 'Exception: {}'.format(device_id, data, except_msg)) return 1
def public_metrics_into_influxdb(): ping, download, upload = get_internet_measure() current_time = get_time() net_json_body = [ { "measurement": "internet_measure", "tags": { "host": "desktop-lab", "contract": "Vivo Fibra" }, "time": str(current_time), "fields": { "ping": ping, "download":download, "upload": upload } } ] print ("\n=========" ) print ("Inserting: \n" + str(net_json_body)) influxdb_host = getting_env_container_hosts() client = InfluxDBClient(influxdb_host, 8086, 'root', 'root', 'internet_measure') client.create_database('internet_measure') client.write_points(net_json_body) print ("=========\n")
def send_metrics_to_influxdb(plugin_config, logger): influx_config = plugin_config['influxdb'] influx_host = influx_config['host'] influx_port = influx_config['port'] influx_database = influx_config['database'] influx_user = influx_config['user'] influx_password = influx_config['password'] tags = influx_config['tags'] tags['influxdb_database'] = influx_database client = InfluxDBClient(influx_host, influx_port, influx_user, influx_password, influx_database) influx_data = {} influx_data['measurement'] = args.task influx_data['tags'] = tags influx_data['fields'] = dict((k, float(v)) for k, v in maas_common.TELEGRAF_METRICS['variables'] .iteritems()) logger.debug("{} - writing metrics to influxdb database " "'{}' at {}".format(args.task, influx_database, influx_host + ':' + str(influx_port))) client.write_points([influx_data]) logger.debug("{} - sent influx_data: {}".format(args.task, influx_data))
def test_query(self): expected = """[{"name":"foo","columns":["time","sequence_number","column_one"],"points":[[1383876043,16,"2"],[1383876043,15,"1"],[1383876035,14,"2"],[1383876035,13,"1"]]}]""" with patch.object(requests, "get") as mocked_get: mocked_get.return_value = _build_response_object(status_code=200, content=expected) cli = InfluxDBClient("host", 8086, "username", "password", "db") result = cli.query("select column_one from foo;") assert len(json.loads(result)[0]["points"]) == 4
def insert_datapoints(datapoints): dbclient = InfluxDBClient(db_server, db_port, db_admin, db_admin_password) dbclient.switch_database(db_name) logger.info('Inserting into database the following datapoints:') logger.info(pformat(datapoints)) response = dbclient.write_points(datapoints)
def init(self): """Init the connection to the InfluxDB server.""" if not self.export_enable: return None try: db = InfluxDBClient(host=self.host, port=self.port, username=self.user, password=self.password, database=self.db) get_all_db = [i['name'] for i in db.get_list_database()] self.version = INFLUXDB_09PLUS except InfluxDBClientError: # https://github.com/influxdb/influxdb-python/issues/138 logger.info("Trying fallback to InfluxDB v0.8") db = InfluxDBClient08(host=self.host, port=self.port, username=self.user, password=self.password, database=self.db) get_all_db = [i['name'] for i in db.get_list_database()] self.version = INFLUXDB_08 except InfluxDBClientError08 as e: logger.critical("Cannot connect to InfluxDB database '%s' (%s)" % (self.db, e)) sys.exit(2) if self.db in get_all_db: logger.info( "Stats will be exported to InfluxDB server: {}".format(db._baseurl)) else: logger.critical("InfluxDB database '%s' did not exist. Please create it" % self.db) sys.exit(2) return db
class InfluxDbConnector(DatabaseConnector): DATABASE_ENGINE_NAME = "InfluxDB" def __init__(self, conf): super(InfluxDbConnector, self).__init__(conf) def _get_section_name(self): return InfluxDbConnector.DATABASE_ENGINE_NAME def connect(self): self._dbClient = InfluxDBClient( self._hostAddress, self._hostPort, self._userName, self._userPassword, self._databaseName ) def query_data(self, query): try: return self._dbClient.query(query) except InfluxDBServerError as e: raise DatabaseConnectionException("InfluxDB has returned 5XX HTTP error code", e) except InfluxDBClientError as e: print DatabaseConnectionException("Unable to send request to InfluxDB", e) def write_data(self, json_data_to_write): try: return self._dbClient.write_points(json_data_to_write) except InfluxDBServerError as e: print DatabaseConnectionException("InfluxDB has returned 5XX HTTP error code", e) except InfluxDBClientError as e: print DatabaseConnectionException("Unable to send request to InfluxDB", e)
def test_get_database_list(self): with patch.object(session, 'get') as mocked_get: mocked_get.return_value = _build_response_object( status_code=200, content='[{"name": "a_db"}]') cli = InfluxDBClient('host', 8086, 'username', 'password') assert len(cli.get_database_list()) == 1 assert cli.get_database_list()[0]['name'] == 'a_db'
def main(): arg = docopt(""" Usage: audioInputLevelsPulse.py [-v] --source=<name> --source=<name> pulseaudio source name (use `pactl list sources | grep Name`) -v Verbose """) log.setLevel(logging.DEBUG if arg['-v'] else logging.INFO) # todo move this into the PeakMonitor part subprocess.check_output(['pactl', 'set-source-volume', arg['--source'], '94900']) influx = InfluxDBClient('bang.vpn-home.bigasterisk.com', 9060, 'root', 'root', 'main') hostname = socket.gethostname() METER_RATE = 8192 monitor = PeakMonitor(arg['--source'], METER_RATE) for sample in monitor: log.debug(' %6.3f %s', sample['value'], '>' * int(min(80, sample['value'] * 80))) influx.write_points([{'measurement': 'audioLevel', "fields": sample, "time": int(time.time())}], tags=dict(location=hostname), time_precision='s')
def main(host='localhost', port=8086, metric_name='KS', num_clients=4): print "influxDB verification start-------------" print "LINE PROTOCOL" user = '******' password = '******' db_name = 'monasca' client = InfluxDBClient(host, port, user, password, db_name) db_user = '******' db_user_password = '******' print "Switch user: {}".format(db_user) client.switch_user(db_user, db_user_password) total_measurements = 0 for i in xrange(1, num_clients + 1): measurements_per_client = 0 for j in xrange(NUMBER_OF_UNIQUE_METRICS): query = 'SELECT count(value) FROM /metric_{0}_{1}_{2}/'.format(metric_name, i, j) result = client.query(query) metric_point = list(result.get_points(measurement='metric_{0}_{1}_{2}'.format( metric_name, i, j))) count_result = metric_point[0]['count'] measurements_per_client += count_result print "{0} measurements per client # {1} = {2}".format( metric_name, i, measurements_per_client) total_measurements += measurements_per_client print "total {0} measurements = {1}".format(metric_name, total_measurements)
def get_metric(request): if request.is_ajax(): if request.method == 'GET': try: db_id = int(request.GET.get('db')) time_from = int(request.GET.get('from')) time_to = int(request.GET.get('to')) metric = request.GET.get('metric') except ValueError: return HttpResponseBadRequest() if db_id is None or time_from is None or time_to is None or metric is None: return HttpResponseBadRequest() try: Database.objects.get(id=db_id, user=request.user) except Database.DoesNotExist: return JsonResponse({'status': 'bad'}) # connect to Influx client = InfluxDBClient( settings.INFLUX_HOST, settings.INFLUX_PORT, settings.INFLUX_USER, settings.INFLUX_PASS, settings.INFLUX_DB ) result = client.query( "select * from metrics where metric='%s' and time > %d and time < %d and \"database\" = %d" % (metric, time_from, time_to, db_id)) return JsonResponse(list(result.get_points(measurement='metrics')), safe=False) else: return HttpResponseBadRequest() else: return HttpResponseBadRequest()
def get(self, node_id, sensor_id, value_type): sensor_node = db.session.query(SensorNode).filter(SensorNode.api_id == node_id).first() value_type_id = value_types.get_id(value_type) if value_type_id is None: return {"message": "Not found"}, 404 value_name = value_types.get_short_name(value_type_id) query = "SELECT value FROM {value_name} WHERE ni='{node_id:d}' AND si='{sensor_id:d}'".format( node_id=sensor_node.id, sensor_id=sensor_id, value_name=value_name ) from influxdb import InfluxDBClient client = InfluxDBClient( host="localhost", port=8086, database="metrics" ) result = client.query(query) values = [] for p in result.get_points(): values.append({ "date": p["time"], "value": p["value"] }) return { "values": values }
app = Flask(__name__) #below for flask-sijax app.config['SIJAX_STATIC_PATH'] = path app.config['SIJAX_JSON_URI'] = '/static/js/sijax/json2.js' flask_sijax.Sijax(app) dict_cnameIDSCnames = {} with open('cname_id_name1.json') as f: dict_cnameIDSCnames = json.load(f) data_cname_sedc = {} with open('data_cname.json') as f: data_cname_sedc = json.load(f) client = InfluxDBClient(host="localhost", port=8086) client.get_list_database() client.switch_database("sedc_test") #get all sedc measurements measurements = client.query("Show measurements").raw series = measurements['series'] m_vals = np.array(series[0]['values']).flatten() ori_response = {} ori_response["mvals"] = m_vals ori_response["cnames_all"] = dict_cnameIDSCnames.keys() def get_TV(data, color): arr = np.array(data['series'][0]['values']) arr_vals = arr[:, -1]
class InfluxReport(object): def __init__(self, global_results, other_results, created_jira_tickets, default_config): self.host = default_config['influx']['host'] self.port = default_config['influx'].get('port', 8086) self.db = default_config['influx'].get('db', 'prodsec') self.login = default_config['influx'].get('login', '') self.policy = default_config['influx'].get('policy', {'Blocker': 1, 'Critical': 5, 'Major': 15}) self.password = default_config['influx'].get('password', '') self.project_name = default_config['project_name'] self.environment = default_config['environment'] self.test_type = default_config['test_type'] self.created_jira_tickets = created_jira_tickets self.open_issues = len(self.created_jira_tickets) self.execution_time = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') self.results = global_results self.results_by_severity = self.sort_results_by_severity() self.results_by_severity['new_in_jira'] = self.get_new_jira_issues() self.results_by_severity['total_in_jira'] = self.open_issues self.results_by_severity['test_to_count'] = 1 self.build_id = f'{self.execution_time} - {self.project_name}' self.client = InfluxDBClient(self.host, self.port, username=self.login, password=self.password, database=self.db) self._ingest_active_errors() def sort_results_by_severity(self): results = dict() for each in self.results: priority = define_jira_priority(each.finding['severity']) if priority not in results: results[priority] = 0 results[priority] += 1 return results def get_new_jira_issues(self): i = 0 for issue in self.created_jira_tickets: i += 1 if issue['new'] else 0 return i def sort_results_by_issue_type(self): # TODO: implement with canonical issue naming pass def out_of_compliance_issues(self): # TODO: implement with compliance policy pass def _ingest_active_errors(self): stats = [{ "measurement": "stats", "time": self.execution_time, "tags": { 'build_id': self.build_id, 'test_name': self.test_type, 'type': self.test_type, 'project': self.project_name }, "fields": self.results_by_severity }] self.client.write_points(stats) jira_issues = [] for issue in self.created_jira_tickets: ts = int(datetime.datetime.strptime(issue['open_date'], '%Y-%m-%dT%H:%M:%S.%f%z').timestamp()) break_policy = 'Y' if str(issue['priority']) in self.policy and \ ts + (self.policy[str(issue['priority'])]*24*3600) < int(time()) else 'N' issue = { "measurement": "errors", "time": self.execution_time, "tags": { 'build_id': self.build_id, 'description': str(issue['description']), 'test_name': self.test_type, 'type': self.test_type, 'project': self.project_name, 'priority': issue['priority'], 'created': datetime.datetime.strptime( issue['open_date'], '%Y-%m-%dT%H:%M:%S.%f%z').strftime('%d %b %Y %H:%M:%S.%f'), 'link': str(issue['link']) }, "fields": { 'breaking_policy': break_policy, 'status': str(issue['status']), 'assignee': str(issue['assignee']), 'quantity': 1 } } jira_issues.append(issue) self.client.write_points(jira_issues)
if __name__ == "__main__": i = 1.1 while (True): json_body = [{ "measurement": "sensorData", "tags": { "source": "bedroom", "type": "temperature" }, "fields": { "value": i } }] i = i * 1.1 if (i >= 100.0): i = 1.1 client = InfluxDBClient('localhost', 8086, 'admin', 'admin', 'hackaday') client.create_database('hackaday') client.write_points(json_body) result = client.query('select value from sensorData;') print("Result: {0}".format(result)) time.sleep(1)
class AlarmOBot: def __init__(self, args): p = argparse.ArgumentParser(prog="alarmobot") p.add_argument("--ngdp-bin", required=True) p.add_argument("--ngdp-dir", required=True) # dir that contains .ngdp p.add_argument("--logfile") p.add_argument("--webhook-url", nargs="*") p.add_argument("--influx-url", nargs="?") p.add_argument("--simulate-new-build", action="store_true") p.add_argument("--from-email", nargs="?", default="root@localhost") p.add_argument("--to-email", nargs="*") p.add_argument("--post-url", nargs="*") p.add_argument("-v", "--verbose", action="store_true") self.args = p.parse_args(args) # Example url: # https://user:[email protected]:8086/dbname influx_url = self.args.influx_url if influx_url: url = urlparse(influx_url) self.influx = InfluxDBClient( host=url.hostname, port=url.port, username=url.username, password=url.password, database=url.path.lstrip("/"), ssl=url.scheme == "https", verify_ssl=url.scheme == "https", timeout=3, ) else: self.influx = None if self.args.to_email and self.args.from_email: self.ses = boto3.client("ses") self.simulate_new_build = self.args.simulate_new_build self.check_count = 0 if self.args.logfile: logging.basicConfig(filename=self.args.logfile) else: logging.basicConfig() self.logger = logging.getLogger("alarmobot") loglevel = logging.DEBUG if self.args.verbose else logging.INFO self.logger.setLevel(loglevel) self.log_buffer = DequeAdapter([], 10) self.logger.addHandler(QueueHandler(self.log_buffer)) self.mention = "" if self.simulate_new_build else "@everyone" def call_proc(self, args, log_stdout=False, log_stderr=False): log_args = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) if log_stdout and log_stderr: log_args["stderr"] = subprocess.STDOUT proc = subprocess.Popen(args, **log_args) while True: output = "" if log_stdout: output = proc.stdout.readline().decode() elif log_stderr: output = proc.stderr.readline().decode() if not output: if proc.poll() is not None: break else: self.logger.debug(output.strip()) return proc def call_ngdp(self, args): ngdp_dir = os.path.join(self.args.ngdp_dir, ".ngdp") return self.call_proc([ self.args.ngdp_bin, "--ngdp-dir", ngdp_dir, "--no-progress", *args ], log_stdout=True, log_stderr=True) def write_to_influx(self, buildinfo): if not self.influx: return self.logger.debug("Writing buildinfo %r to InfluxDB", buildinfo) result = self.influx.write_points([{ "measurement": "hsb_build", "tags": { "build": buildinfo }, "fields": { "count": 1 }, "time": datetime.now().isoformat(), }]) if not result: self.logger.warning("Error writing points to InfluxDB") def write_to_discord(self, message, tts=False): if self.args.webhook_url: for webhook_url in self.args.webhook_url: self.logger.debug("Sending to Discord: %r", message) resp = requests.post(webhook_url, json={ "content": message, "tts": tts }) self.logger.debug("Response: %r", resp) def send_email(self, message): if not self.args.to_email or not self.args.from_email: return self.logger.debug("Attempting to send email") try: self.ses.send_email( Source="Alarm-o-Bot <{}>".format(self.args.from_email), Destination={"ToAddresses": self.args.to_email}, Message={ "Subject": { "Charset": "UTF-8", "Data": "Hearthstone build data updated", }, "Body": { "Text": { "Charset": "UTF-8", "Data": message.replace("\n", "\r\n"), } } }) except Exception: self.logger.exception("Exception while sending email") def compare_versions(self, old_version, new_version): if not old_version or not old_version.versions_name: raise ValueError("old_version is not a valid version object: %s" % old_version) if not new_version or not new_version.versions_name: raise ValueError("new_version is not a valid version object: %s" % new_version) self.check_count += 1 if old_version.versions_name != new_version.versions_name: return True if self.simulate_new_build and self.check_count == 1: self.logger.info("Simulating a new build arriving") self.simulate_new_build = False return True return False def on_new_build(self, old, new): # Send an alert on Influx self.write_to_discord(f"{self.mention} Hearthstone build data updated", tts=True) message = MESSAGE.format(mention=self.mention, old=old.versions_name, new=new.versions_name) self.write_to_discord(message) # Send emails self.send_email(message) if self.args.post_url: for url in self.args.post_url: requests.post(url) out_dir = os.path.join(self.args.ngdp_dir, new.build_id) # Start downloading the patch self.write_to_discord("Downloading patch…") self.logger.info("Downloading…") ngdp_proc = self.call_ngdp(["fetch", "hsb"]) ngdp_proc.wait() if ngdp_proc.returncode != 0: error = "\n".join(map(lambda lr: lr.getMessage(), self.log_buffer)) self.write_to_discord( f"{self.mention} Patch download failed: ```{error}```") return self.write_to_discord( f"Successfully downloaded new build, installing to `{out_dir}`…") ngdp_proc = self.call_ngdp( ["install", "hsb", new.build_config, out_dir]) ngdp_proc.wait() if ngdp_proc.returncode != 0: error = "\n".join(map(lambda lr: lr.getMessage(), self.log_buffer)) self.write_to_discord( f"{self.mention} Patch installation failed: ```{error}```") else: self.write_to_discord( f"Successfully installed new build to `{out_dir}`!") def get_latest_version(self): remote = HttpRemote("http://us.patch.battle.net:1119/hsb") try: versions = remote.get_versions() except Exception: return None versions = [v for v in versions if v.region == "us"] return max(versions, key=lambda x: x.build_id) def check_for_new_version(self, current_version): new_version = self.get_latest_version() if not new_version: self.logger.warning("Got invalid version, skipping heartbeat") return current_version if self.compare_versions(current_version, new_version): self.logger.info("New build: %s", new_version.versions_name) self.on_new_build(current_version, new_version) current_version = new_version self.write_to_influx(current_version.versions_name) return current_version def run(self): try: version = self.get_latest_version() if not version or not version.versions_name: raise RuntimeError("Unable to get current version") self.logger.info("Current build: %s", version.versions_name) while True: version = self.check_for_new_version(version) time.sleep(5) except KeyboardInterrupt: pass return 0
"Luminance": Sensor_Read.Luminance, "RelativeHumidity": Sensor_Read.RelativeHumidity, "Temperature": Sensor_Read.Temperature } }] print("Dinesh" + str(json_body)) client_influx.write_points(json_body) print("Wrote values to DB") update_problem_pddl() print("Problem Updated") pddl_plan = get_pddl_plan() publish_pddl_plan(pddl_plan) print(pddl_plan) client_influx = InfluxDBClient('localhost', 8086, 'root', 'root', 'Sensor_Values') client_influx.create_database('Sensor_Values') #MQTT Client instantiation mqtt_client = mqtt.Client() # Linking the client with callback implementations mqtt_client.on_connect = on_connect mqtt_client.on_message = on_message # Connecting to the MQTT Broker mqtt_client.connect(MQTT_BROKER_IP, MQTT_BROKER_PORT, MQTT_KEEPALIVE_INTERVAL) mqtt_client.loop_start() mqtt_client.subscribe(MQTT_TOPIC) print("Subscribed !!")
from influxdb import InfluxDBClient import time, math, random #获取当前运行的pid p1 = psutil.Process(os.getpid()) from influxdb import InfluxDBClient import time, math, random while True: a = psutil.virtual_memory().percent #内存占用率 b = psutil.cpu_percent(interval=1.0) #cpu占用率 json_body = [{ "measurement": "cpu_load_short", "tags": { "host": "server01", "region": "us-west" }, #"time": "2009-11-10T23:00:00Z", "fields": { "cpu": b, "mem": a } }] client = InfluxDBClient('localhost', 8086, 'root', 'root', 'xxyyxx') client.create_database('xxyyxx', if_not_exists=False) client.write_points(json_body) #result = client.query('select value from cpu_load_short;') #print("Result: {0}".format(result)) time.sleep(2)
# snmp constants HOSTNAME = 'localhost' COMMUNITY = 'public' OID_RAM_UNUSED = 'iso.3.6.1.4.1.2021.4.11.0' OID_DISK_USAGE = 'iso.3.6.1.4.1.2021.9.1.9.1' OID_CPU_USAGE = 'iso.3.6.1.2.1.25.3.3.1.2' # ...1.2.core_number URL_DASHBOARD = 'http://localhost:8888' DB_NAME = 'system' GRAPHICS_PATH = './Analytics' # Initializing snmp session session = Session(hostname=HOSTNAME, community=COMMUNITY, version=2) # Initializing database client = InfluxDBClient(HOSTNAME, 8086, 'admin', '1234', DB_NAME) client.create_database(DB_NAME) # Open dashboard automatically web.open_new_tab(URL_DASHBOARD) print('Running...') print('Press ctrl+c to stop monitoring and view results.') while True: entry = [] cpus = session.walk(OID_CPU_USAGE) nCores = len(cpus) load_sum = 0 for c in cpus: load_sum += int(c.value)
bConfig = CONFDIR + args.profile #logger.info(bConfig) config.read(bConfig) tbls = config['TIAnalysis']['scantbls'] logmode = config['scan']['debugMode'] host = config['influx']['host'] port = config['influx']['port'] password = config['influx']['passwd'] dbnameBF = config['influx']['dbnameBF'] seriesBF = config['influx']['seriesBF'] user = config['influx']['user'] durationStr = config['backfill']['durationStr'] barSizeSetting = config['backfill']['barSizeSetting'] backdays = int(config['backfill']['backdays']) myclient = InfluxDBClient(host, port, user, password, dbnameBF) myclient.create_database(dbnameBF) """ Modelled (loosely) on example here: https://docs.python.org/3/library/asyncio-queue.html#examples and here: https://realpython.com/async-io-python/#using-a-queue """ class MySeriesHelper(SeriesHelper): """Instantiate SeriesHelper to write points to the backend.""" class Meta: """Meta class stores time series helper configuration.""" # The client should be an instance of InfluxDBClient.
import pandas as pd from influxdb import InfluxDBClient client = InfluxDBClient(host='floodnet-influxdb.sonycproject.com', ssl=True, port=0, database='db0') measure = 'distance' table = 'msg' app_id = 'deployment_one_app' dev_id = 'sensor_2' days_back = 7 query = 'SELECT "%s" ' \ 'FROM "%s" ' \ 'WHERE "app_id" = \'%s\' ' \ 'AND "dev_id" = \'%s\' ' \ 'AND "time" >= now() - %id'\ % (measure, table, app_id, dev_id, days_back) data = client.query(query) value_data = data.raw['series'][0]['values'] df = pd.DataFrame(value_data, columns=['datetime', 'distance']) df['datetime'] = pd.to_datetime(df['datetime'], format='%Y-%m-%dT%H:%M:%S.%f') df['localtime'] = df['datetime'].dt.tz_convert('US/Eastern') print(df)
def getGardenCallback(self, err, res): if err: print err else: if configuration["influxdb"]["enabled"]: influxDbClient = InfluxDBClient( configuration["influxdb"]["server"], configuration["influxdb"]["port"], configuration["influxdb-username"], configuration["influxdb"]["password"], configuration["influxdb"]["database"]) try: influxDbClient.create_database( configuration["influxdb"]["database"]) except InfluxDBClientError, ex: print "InfluxDBClientError", ex influxDbClient.create_retention_policy( configuration["influxdb"]["policy"], 'INF', 3, default=True) for location in res["locations"]: #print json.dumps(location, indent=2, sort_keys=True) sensorId = location["sensor"]["sensor_identifier"][-4:].lower() flower = {} #flower["sensor_name"] = location["sensor"]["sensor_identifier"] if location["battery"]["gauge_values"][ "current_value"] is not None: flower["battery"] = ( "Battery", int(location["battery"]["gauge_values"] ["current_value"])) if location["air_temperature"]["gauge_values"][ "current_value"] is not None: flower["air_temperature"] = ( "Temperature", float(location["air_temperature"]["gauge_values"] ["current_value"])) flower["air_temperature_status"] = [ "Temperature Status", str(location["air_temperature"] ["instruction_key"]).replace( "air_temperature_", ""), ["good", "too_low", "too_high"] ] if location["fertilizer"]["gauge_values"][ "current_value"] is not None: flower["fertilizer"] = ( "Fertilizer", float(location["fertilizer"]["gauge_values"] ["current_value"])) flower["fertilizer_status"] = [ "Fertilizer Status", str(location["fertilizer"]["instruction_key"]).replace( "fertilizer_", ""), ["good", "too_low", "too_high"] ] if location["light"]["gauge_values"][ "current_value"] is not None: flower["light"] = ("Light", float(location["light"]["gauge_values"] ["current_value"])) flower["light_status"] = [ "Light Status", str(location["light"]["instruction_key"]).replace( "light_", ""), ["good", "too_low", "too_high"] ] if location["watering"]["soil_moisture"]["gauge_values"][ "current_value"] is not None: flower["watering"] = ( "Moisture", float(location["watering"]["soil_moisture"] ["gauge_values"]["current_value"])) flower["watering_status"] = [ "Moisture Status", str(location["watering"]["soil_moisture"] ["instruction_key"]).replace("soil_moisture_", ""), ["good", "too_low", "too_high"] ] lastUtc = ("Updated", str(location["last_sample_utc"])) if configuration["mqtt"]["enabled"]: print "Pushing Mqtt", sensorId, ":", configuration["mqtt"][ "prefix"], flower try: broadcastMqtt(configuration["mqtt"]["client"], configuration["mqtt"]["server"], configuration["mqtt"]["port"], configuration["mqtt"]["prefix"], sensorId + "/update", json.dumps(taflowerg)) except Exception, ex: print "Error on mqtt broadcast", ex if configuration["prometheuspush"]["enabled"]: registry = CollectorRegistry() for key in flower.keys(): print "Pushing", sensorId, ":", configuration[ "prometheuspush"][ "prefix"] + '_' + key + '_total', "=", flower[ key] if flower[key][1] is None: continue elif type(flower[key][1]) is str: e = Enum( configuration["prometheuspush"]["prefix"] + '_' + key + '_total', flower[key][0], ['sensorid'], states=flower[key][2], registry=registry) e.labels(sensorid=sensorId).state(flower[key][1]) else: g = Gauge( configuration["prometheuspush"]["prefix"] + '_' + key + '_total', flower[key][0], ['sensorid'], registry=registry) g.labels(sensorid=sensorId).set(flower[key][1]) print "Pushing", sensorId, ":", configuration[ "prometheuspush"][ "prefix"] + '_' + key + '_total', "=", flower[key] try: push_to_gateway( configuration["prometheuspush"]["server"] + ":" + configuration["prometheuspush"]["port"], job=configuration["prometheuspush"]["client"] + "_" + sensorId, registry=registry) except: print "Prometheus not available" if configuration["influxdb"]["enabled"]: influxDbJson = [{ "measurement": configuration["influxdb"]["prefix"], "tags": { "sensor": sensorId, }, "time": lastUtc[1], "fields": {} }] for key in flower.keys(): influxDbJson[0]["fields"][key] = flower[key][1] print "Pushing", influxDbJson try: influxDbClient.write_points( influxDbJson, retention_policy=configuration["influxdb"] ["policy"]) except: print "Influxdb not available"
class Output(cowrie.core.output.Output): """ influx output """ def start(self): host = CowrieConfig.get("output_influx", "host", fallback="") port = CowrieConfig.getint("output_influx", "port", fallback=8086) ssl = CowrieConfig.getboolean("output_influx", "ssl", fallback=False) self.client = None try: self.client = InfluxDBClient(host=host, port=port, ssl=ssl, verify_ssl=ssl) except InfluxDBClientError as e: log.msg(f"output_influx: I/O error({e.code}): '{e.message}'") return if self.client is None: log.msg("output_influx: cannot instantiate client!") return if CowrieConfig.has_option("output_influx", "username") and CowrieConfig.has_option( "output_influx", "password"): username = CowrieConfig.get("output_influx", "username") password = CowrieConfig.get("output_influx", "password", raw=True) self.client.switch_user(username, password) try: dbname = CowrieConfig.get("output_influx", "database_name") except Exception: dbname = "cowrie" retention_policy_duration_default = "12w" retention_policy_name = dbname + "_retention_policy" if CowrieConfig.has_option("output_influx", "retention_policy_duration"): retention_policy_duration = CowrieConfig.get( "output_influx", "retention_policy_duration") match = re.search(r"^\d+[dhmw]{1}$", retention_policy_duration) if not match: log.msg( ("output_influx: invalid retention policy." "Using default '{}'..").format(retention_policy_duration)) retention_policy_duration = retention_policy_duration_default else: retention_policy_duration = retention_policy_duration_default database_list = self.client.get_list_database() dblist = [str(elem["name"]) for elem in database_list] if dbname not in dblist: self.client.create_database(dbname) self.client.create_retention_policy( retention_policy_name, retention_policy_duration, 1, database=dbname, default=True, ) else: retention_policies_list = self.client.get_list_retention_policies( database=dbname) rplist = [str(elem["name"]) for elem in retention_policies_list] if retention_policy_name not in rplist: self.client.create_retention_policy( retention_policy_name, retention_policy_duration, 1, database=dbname, default=True, ) else: self.client.alter_retention_policy( retention_policy_name, database=dbname, duration=retention_policy_duration, replication=1, default=True, ) self.client.switch_database(dbname) def stop(self): pass def write(self, entry): if self.client is None: log.msg("output_influx: client object is not instantiated") return # event id eventid = entry["eventid"] # measurement init m = { "measurement": eventid.replace(".", "_"), "tags": { "session": entry["session"], "src_ip": entry["src_ip"] }, "fields": { "sensor": self.sensor }, } # event parsing if eventid in ["cowrie.command.failed", "cowrie.command.input"]: m["fields"].update({ "input": entry["input"], }) elif eventid == "cowrie.session.connect": m["fields"].update({ "protocol": entry["protocol"], "src_port": entry["src_port"], "dst_port": entry["dst_port"], "dst_ip": entry["dst_ip"], }) elif eventid in ["cowrie.login.success", "cowrie.login.failed"]: m["fields"].update({ "username": entry["username"], "password": entry["password"], }) elif eventid == "cowrie.session.file_download": m["fields"].update({ "shasum": entry.get("shasum"), "url": entry.get("url"), "outfile": entry.get("outfile"), }) elif eventid == "cowrie.session.file_download.failed": m["fields"].update({"url": entry.get("url")}) elif eventid == "cowrie.session.file_upload": m["fields"].update({ "shasum": entry.get("shasum"), "outfile": entry.get("outfile"), }) elif eventid == "cowrie.session.closed": m["fields"].update({"duration": entry["duration"]}) elif eventid == "cowrie.client.version": m["fields"].update({ "version": ",".join(entry["version"]), }) elif eventid == "cowrie.client.kex": m["fields"].update({ "maccs": ",".join(entry["macCS"]), "kexalgs": ",".join(entry["kexAlgs"]), "keyalgs": ",".join(entry["keyAlgs"]), "compcs": ",".join(entry["compCS"]), "enccs": ",".join(entry["encCS"]), }) elif eventid == "cowrie.client.size": m["fields"].update({ "height": entry["height"], "width": entry["width"], }) elif eventid == "cowrie.client.var": m["fields"].update({ "name": entry["name"], "value": entry["value"], }) elif eventid == "cowrie.client.fingerprint": m["fields"].update({"fingerprint": entry["fingerprint"]}) # cowrie.direct-tcpip.data, cowrie.direct-tcpip.request # cowrie.log.closed # are not implemented else: # other events should be handled log.msg( f"output_influx: event '{eventid}' not handled. Skipping..") return result = self.client.write_points([m]) if not result: log.msg("output_influx: error when writing '{}' measurement" "in the db.".format(eventid))
# Slow as crap, but worth holding on to for project comparison. import pandas as pd from influxdb import InfluxDBClient bldg = input('bldg ID: ').upper() source = input("'hotIN', 'coldIN', or 'hotRETURN': ") print('locating file...\n') path = "/Users/joseph/Desktop/GRA/InfluxSemesterProject/LLC_BLDG_" + bldg + "/" file = source + "_LLC_BLDG_" + bldg + "_OCT-4-NOV-13_Testdata.csv" print('Connecting to InfluxDB...\n') client = InfluxDBClient(host='influxdbubuntu.bluezone.usu.edu', port=8086) client.switch_database('LLC_FlowData') #path="/Users/joseph/Desktop/GRA/InfluxSemesterProject/" #file = "test.csv" print('Reading CSV file...\n') csvReader = pd.read_csv(path + file, sep=',') print(csvReader.shape) print(csvReader.columns) data = len(csvReader) pbar = ProgressBar(widgets=widgets, maxval=data).start()
class TimeSeriesStore(): def __init__(self, write_interval=20): import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) self.client = InfluxDBClient('influx.fabi.me', 8086, 'tescan', 't3sc4n', 'tescan_mon', ssl=True, retries=1, timeout=10) self.queue = queue.Queue() self.write_interval = write_interval self._write_thread = Thread(target=self._write_loop, daemon=True) self._write_thread.start() self._num_write_errors = 0 self.fallback = FallbackStore() def write(self, measurement, time: datetime.datetime, tags, data): if not data: return False self.queue.put({ "measurement": measurement, "tags": tags, "time": time.isoformat(), "fields": data, }) return True def _write_loop(self): while True: points = [] while not self.queue.empty() and len(points) < 20_000: points.append(self.queue.get()) try: if points: self.client.write_points(points, time_precision='ms') print('tss wrote', len(points), 'points with', sum(map(len, points)), 'fields') self._num_write_errors = 0 except (Exception, IOError) as e: self._num_write_errors += 1 print(type(e), 'error writing', len(points), 'points', e, 'num err', self._num_write_errors) if self._num_write_errors > 2: print('fallback store', len(points), 'points') self.fallback.write(points) else: for p in points: self.queue.put(p) print('re-queued', len(points), 'points, qsize=', self.queue.qsize(), 'ne=', self._num_write_errors) time.sleep(self.write_interval * 4) time.sleep(self.write_interval) def flush(self): points = [] while not self.queue.empty(): points.append(self.queue.get()) print('flushing', len(points), 'points') if points: try: self.client.write_points(points, time_precision='ms') except Exception as e: print('Error %s flush write, using fallback' % e) self.fallback.write(points, sync=True)
def start(self): host = CowrieConfig.get("output_influx", "host", fallback="") port = CowrieConfig.getint("output_influx", "port", fallback=8086) ssl = CowrieConfig.getboolean("output_influx", "ssl", fallback=False) self.client = None try: self.client = InfluxDBClient(host=host, port=port, ssl=ssl, verify_ssl=ssl) except InfluxDBClientError as e: log.msg(f"output_influx: I/O error({e.code}): '{e.message}'") return if self.client is None: log.msg("output_influx: cannot instantiate client!") return if CowrieConfig.has_option("output_influx", "username") and CowrieConfig.has_option( "output_influx", "password"): username = CowrieConfig.get("output_influx", "username") password = CowrieConfig.get("output_influx", "password", raw=True) self.client.switch_user(username, password) try: dbname = CowrieConfig.get("output_influx", "database_name") except Exception: dbname = "cowrie" retention_policy_duration_default = "12w" retention_policy_name = dbname + "_retention_policy" if CowrieConfig.has_option("output_influx", "retention_policy_duration"): retention_policy_duration = CowrieConfig.get( "output_influx", "retention_policy_duration") match = re.search(r"^\d+[dhmw]{1}$", retention_policy_duration) if not match: log.msg( ("output_influx: invalid retention policy." "Using default '{}'..").format(retention_policy_duration)) retention_policy_duration = retention_policy_duration_default else: retention_policy_duration = retention_policy_duration_default database_list = self.client.get_list_database() dblist = [str(elem["name"]) for elem in database_list] if dbname not in dblist: self.client.create_database(dbname) self.client.create_retention_policy( retention_policy_name, retention_policy_duration, 1, database=dbname, default=True, ) else: retention_policies_list = self.client.get_list_retention_policies( database=dbname) rplist = [str(elem["name"]) for elem in retention_policies_list] if retention_policy_name not in rplist: self.client.create_retention_policy( retention_policy_name, retention_policy_duration, 1, database=dbname, default=True, ) else: self.client.alter_retention_policy( retention_policy_name, database=dbname, duration=retention_policy_duration, replication=1, default=True, ) self.client.switch_database(dbname)
# Setup logging log = logging.getLogger('energy-logger') log.setLevel(getattr(logging, loglevel)) if logfile: loghandle = logging.FileHandler(logfile, 'w') formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') loghandle.setFormatter(formatter) else: loghandle = logging.StreamHandler() log.addHandler(loghandle) log.info('Started app') # Create the InfluxDB object influx_config = yaml.load(open('influx_config.yml'), Loader=yaml.FullLoader) client = InfluxDBClient(influx_config['host'], influx_config['port'], influx_config['user'], influx_config['password'], influx_config['dbname']) collector = DataCollector(influx_client=client, meter_yaml=args.meters) repeat(interval, max_iter=collector.max_iterations, func=lambda: collector.collect_and_store())
import time import sys import datetime from influxdb import InfluxDBClient from devicefunctions import * import paho.mqtt.client as mqtt global logfile # Configure InfluxDB connection variables host = "127.0.0.1" # My Ubuntu NUC port = 8086 # default port user = "******" # the user/password created for the pi, with write access password = "******" dbname = "iotlab" # the database we created earlier interval = 5 # Sample period in seconds # Create the InfluxDB client object clientinflux = InfluxDBClient(host, port, user, password, dbname) mqtt_host = "127.0.0.1" mqtt_port = 1883 mqtt_timeinterval = 60 #________mqtt AREA________________________________ def on_connect(client, userdata, flags, rc): print("Fault_tolerant Mechanism started......") logfile.write("Fault_tolerant Mechanism started......" + " at " + str(time.asctime(time.localtime(time.time()))) + "\n") def on_publish(client, userdata, result): return True
record processor. It is not possible to checkpoint once a record processor has lost its lease. * Shutdown Reason TERMINATE: **THE RECORD PROCESSOR MUST CHECKPOINT OR THE KCL WILL BE UNABLE TO PROGRESS** A record processor will be shutdown once it reaches the end of a shard. A shard ending indicates that it has been either split into multiple shards or merged with another shard. To begin processing the new shard(s) it's required that a final checkpoint occurs. :param amazon_kclpy.messages.ShutdownInput shutdown_input: Information related to the shutdown request """ try: if shutdown_input.reason == 'TERMINATE': # Checkpointing with no parameter will checkpoint at the # largest sequence number reached by this processor on this # shard id print('Was told to terminate, will attempt to checkpoint.') self.checkpoint(shutdown_input.checkpointer, None) else: # reason == 'ZOMBIE' print('Shutting down due to failover. Will not checkpoint.') except: pass if __name__ == "__main__": DB_ENDPOINT = os.environ["DB_ENDPOINT"] client = InfluxDBClient(DB_ENDPOINT, 8086, 'admin', 'admin', 'ev') client.create_database('ev') kcl_process = kcl.KCLProcess(RecordProcessor()) kcl_process.run()
iuser = auth['iuser'] ipassword = auth['ipassword'] idbname = auth['idbname'] except: print("Problem loading nextract_config.json") pass if len(hostname) == 0: print("Usage: %s HMC-hostname HMC-username HMC-password" % (sys.argv[0])) sys.exit(1) #print("HMC hostanme=%s User=%s Password=%s" %( hostname, user, password)) if output_influx: from influxdb import InfluxDBClient client = InfluxDBClient(ihost, iport, iuser, ipassword, idbname) print("-> Logging on to %s as user %s" % (hostname, user)) hmc = hmc.HMC(hostname, user, password) print("-> Get Preferences") # returns XML text prefstripped = hmc.get_stripped_preferences_pcm() if debug: hmc.save_to_file("server_perferences.xml", prefstripped) print("-> Parse Preferences") serverlist = hmc.parse_prefs_pcm( prefstripped) # returns a list of dictionaries one per Server perflist = [] all_true = True print("-> ALL servers:")
dir_name = os.path.dirname(os.path.realpath(__file__)) conf = dir_name + '/config.cf' config = ConfigParser.ConfigParser() config.read(conf) meterID = config.get('default', 'meterID') memberID = config.get('default', 'memberID') settlement_point = config.get('default', 'settlement_point') api_url = config.get('default', 'api_url') influxdb_protocol = config.get('influxdb', 'protocol') influxdb_host = config.get('influxdb', 'host') influxdb_port = config.get('influxdb', 'port') influxdb_dbname = config.get('influxdb', 'dbname') client = InfluxDBClient(host=influxdb_host, port=influxdb_port) client.switch_database(influxdb_dbname) payload = { 'meterID': meterID, 'memberID': memberID, 'settlement_point': settlement_point } r = requests.post(api_url, data=json.dumps(payload)) j = json.loads(r.text) influxdb_entry = ( 'electric_cost,price_date={price_date},future_date={future_date} ' 'future_price={future_price},current_price={current_price},' 'value_score={value_score},future_value_score={future_value_score},'
import json import pytz from datetime import datetime from confluent_kafka import Consumer from influxdb import InfluxDBClient # create InfluxDB Connector and create database if not already done client = InfluxDBClient('localhost', 8086, 'root', 'root', 'machinedata') client.create_database('machinedata') result = client.query('select count(*) from machinedata;') print(result)
def influxDBconnect(): """Instantiate a connection to the InfluxDB.""" influxDBConnection = InfluxDBClient(host, port, user, password, dbname) return influxDBConnection
from models.version import VersionModel connect( **{ 'db': 'dms-v2', 'host': None, 'port': None, 'username': os.getenv('MONGO_ID'), 'password': os.getenv('MONGO_PW') }) CLIENT = InfluxDBClient( **{ 'host': 'localhost', 'port': 8086, 'username': '******', 'password': os.getenv('INFLUX_PW_{}'.format('DMS_V2'), 'root'), 'database': 'dms_v2' }) MEASUREMENT = 'version' version_data = {1: '', 2: '', 3: ''} for version in VersionModel.objects: version_data[version.platform] = version.version payload = [{'measurement': MEASUREMENT, 'fields': version_data}] CLIENT.write_points(payload)
#beginDate = "'2019-03-22T12:00:00Z'" #endDate = "'2019-03-24T00:00:00Z'" bldgs = ['B', 'C','D','E','F'] weeks = [1,2,3,4] for bldg in bldgs: for week in weeks: bldgID ="'"+bldg+"'" dates= data_chunk(week) # Retrieve data print('\nConnecting to database...') # Create client object with InfluxDBClient library client = InfluxDBClient(host='odm2equipment.uwrl.usu.edu', port=8086, username='******',password='******') client.switch_database('ciws_final') # Set database. # write query print('Assembling data query...') # Build query by concatenating inputs into query. InfluxDB query language has several # requirements. Fields/Tags must be bracketed with " " and the field/tag values must be bracketed with ' '. # Query returns a 'ResultSet" type. Have to convert to pandas dataframe. query = """SELECT * FROM "LLC" WHERE "buildingID" ="""+bldgID+""" AND time >= """+dates[0]+""" AND time <= """+dates[1]+"""""" print('Retrieving data...') # Query returns a 'ResultSet" type. Have to convert to pandas dataframe. # Convert returned ResultSet to Pandas dataframe with list and get_points. results = client.query(query) # send query df = pd.DataFrame(list(results.get_points(measurement='LLC')))
def __init__(self, dsn): self.client = InfluxDBClient.from_dsn(dsn)
def run(): if len(sys.argv) < 2: print('Please provide the path to the .sumocfg of a SUMO scenario') exit() influx_host = INFLUX_HOST influx_port = INFLUX_PORT sumo_host = SUMO_HOST sumo_port = SUMO_PORT if len(sys.argv) == 6: influx_host = sys.argv[2] influx_port = int(sys.argv[3]) sumo_host = sys.argv[4] sumo_port = int(sys.argv[5]) print('started') client = InfluxDBClient(influx_host, influx_port, 'admin', 'admin') client.drop_database(SUMO_DB) client.create_database(SUMO_DB) client.switch_database(SUMO_DB) print('client available') ## GUI / CLI traci.init(port=sumo_port, numRetries=10, host=sumo_host) print('traci init') step = 0 startTime = datetime.now() print('started at ', startTime.strftime('%Y-%m-%dT%H:%M:%SZ')) while traci.simulation.getMinExpectedNumber() > 0: traci.simulationStep() departed_ids = traci.simulation.getDepartedIDList() # ALL VEHICLES [subscribe(x) for x in departed_ids] # ONLY BUS # [subscribe(x) for x in departed_ids if traci.vehicle.getTypeID(x) == 'bus'] subscription_results = traci.vehicle.getAllSubscriptionResults() vehicles = [subscriberToInfluxJson( id, subscription_results[id]) for id in subscription_results] client.write_points(vehicles) step += 1 if(step % 1000 == 0): print('\n', 'Subscriptions: ', len(subscription_results)) print('Time: ', datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ')) traci.close() endTime = datetime.now() print("Simulation Time:", startTime.strftime('%Y-%m-%dT%H:%M:%SZ'), endTime.strftime('%Y-%m-%dT%H:%M:%SZ'), "differnce:", (endTime - startTime).strftime('%Y-%m-%dT%H:%M:%SZ')) sys.stdout.flush()
def query_to_firebase(): client_influx = InfluxDBClient(host='localhost', port=8086, username='******', password='******') client_influx.switch_database('test_energy') global status, yesterday, date_now json = {} p1, p2, p3, p4 = [], [], [], [] s1, s2, s3, s4 = [], [], [], [] i1, i2, i3, i4 = [], [], [], [] p1_wh, p2_wh, p3_wh, p4_wh = [], [], [], [] time = [] while (status == 1 or status == 2): if(status == 1): results = client_influx.query(("SELECT * FROM %s where time >= '%s' and time <= '%s'") % ('energy_monitor', yesterday, date_now)) points = results.get_points() for item in points: time_obj = parse(item['time']) unixtime = (calendar.timegm(time_obj.timetuple())) time.append(unixtime) p1.append(item['P1']) p2.append(item['P2']) p3.append(item['P3']) p4.append(item['P4']) s1.append(item['S1']) s2.append(item['S2']) s3.append(item['S3']) s4.append(item['S4']) i1.append(item['I1']) i2.append(item['I2']) i3.append(item['I3']) i4.append(item['I4']) p1_wh.append(item['P1_wh']) p2_wh.append(item['P2_wh']) p3_wh.append(item['P3_wh']) p4_wh.append(item['P4_wh']) data = {"time":time, "P1": p1, "P2": p2, "P3": p3, "P4": p4, "S1": s1, "S2": s2, "S3": s3, "S4": s4, "I1": i1, "I2": i2, "I3": i3, "I4": i4, "P1_wh": sum(p1_wh), "P2_wh": sum(p2_wh), "P3_wh": sum(p3_wh), "P4_wh": sum(p4_wh) } json[yesterday] = data print("query Ja") print(yesterday, date_now) print("finished query") query = True if(query): try: print(json) print("uploaded to firebase") status = 0 yesterday = "" date_now = "" query = False except: status = 2 time.sleep(10)
import os import sys import time import urllib2 import teslajson import logging from influxdb import InfluxDBClient from pprint import pprint from config import * a_vin = "" a_displayname = "" a_ignore = ["media_state", "software_update", "speed_limit_mode"] influxclient = InfluxDBClient(a_influxhost, a_influxport, a_influxuser, a_influxpass, a_influxdb) def setup_custom_logger(name): formatter = logging.Formatter( fmt='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') handler = logging.FileHandler(a_logfile, mode='a') handler.setFormatter(formatter) screen_handler = logging.StreamHandler(stream=sys.stdout) screen_handler.setFormatter(formatter) logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) logger.addHandler(handler) logger.addHandler(screen_handler) return logger
from twython import Twython from influxdb import InfluxDBClient dbName = 'testdb3' db = InfluxDBClient(host='localhost', port=8086) db.switch_database(dbName) from auth import (consumer_key, consumer_secret, access_token, access_token_secret) twitter = Twython(consumer_key, consumer_secret, access_token, access_token_secret) def Tweet(image, message): image = open('/home/pi/Documents/BasilBoys/%s.jpg' % image, 'rb') response = twitter.upload_media(media=image) media_id = [response['media_id']] twitter.update_status(status=message, media_ids=media_id) print("Tweeted: %s" % message)
print('Opening up serial connection to', port) connection = serial.Serial(port=port, baudrate=115200, timeout=10) np.set_printoptions(formatter={'float': lambda f: '%5.02f' % f}) position = np.tile(points.mean(axis=0), [10, 1]) current_point, = plt.plot(position[-1, 0], position[-1, 1], 's') ellipse = patches.Ellipse(points.mean(axis=0), 1, 1, alpha=0.3, color='r') plt.gca().add_patch(ellipse) dist_circles = [] for p in points: c = patches.Circle(p, 1, fill=None, alpha=0.4) dist_circles.append(c) ax.add_patch(c) client = InfluxDBClient(args.host, args.port) # client.create_database(args.database) client.switch_database(args.database) # client.create_retention_policy('stream_rp', '52w', 1, default=True) tags = {'sourceId': args.sourceId} try: while True: line = connection.readline() if not line.startswith('mc'): continue try: fields = line.split() dists = np.array(map(lambda x: int(x, 16) / 1000.0, fields[2:5])) for i in range(len(dists)):
global run global config logger.info("Exiting.") client.loop_stop() run = False for d_k, d in config.items(): d["online"] = False db_write_point(d_k, "status", 0) save_config_json(config) run = True signal.signal(signal.SIGINT, sig_exit) signal.signal(signal.SIGTERM, sig_exit) config = load_config_json() db = InfluxDBClient(DB_SERVER, DB_PORT, database=DB_NAME) try: db.ping() client = mqtt.Client() client.enable_logger(logger) client.on_connect = on_connect client.on_message = on_message client.connect(MQTT_SERVER, MQTT_PORT, 60) client.loop_start() while run: online_devices = {k: v for k, v in config.items() if v["online"]}