def reportMetrics(pool): p = redis.StrictRedis(connection_pool=pool).pipeline(transaction=False) opentsdbClient = potsdb.Client(OPENTSDB_URL) while True: time.sleep(REPORT_INTERVAL * 60) lastWindow = getTimestampByMinute(datetime.utcnow() - timedelta(minutes=REPORT_DELAY)) p.zrangebyscore('TS', 0, lastWindow) p.zremrangebyscore('TS', 0, lastWindow) tsRange = p.execute() metricsCounter = {} for tsKey in tsRange[0]: p.hgetall(tsKey) results = p.execute() for tsKey, result in zip(tsRange[0], results): metricsCounter[tsKey] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] for key, value in result.iteritems(): index = COUNTER_KEYS[key] metricsCounter[tsKey][index] = int(value) p.hdel(tsKey, key) p.execute() reportOpenTsdb(metricsCounter, opentsdbClient)
def get_db_client(self): return potsdb.Client(host=settings.OPENTSDB_HOST, port=settings.OPENTSDB_PORT, qsize=1000, host_tag=True, mps=100, check_host=True)
def reportMetrics(pool): p = redis.StrictRedis(connection_pool=pool).pipeline(transaction=False) opentsdbClient = potsdb.Client(OPENTSDB_URL) while True: time.sleep(REPORT_INTERVAL * 60) lastWindow = getTimestampByMinute(datetime.utcnow() - timedelta(minutes=REPORT_DELAY)) p.zrangebyscore('TS', 0, lastWindow) p.zremrangebyscore('TS', 0, lastWindow) tsRange = p.execute() metricsCounter = {} for tsKey in tsRange[0]: p.get(tsKey) results = p.execute() for tsKey, result in zip(tsRange[0], results): #logger.info("tsKey:%s, result:%s" % (tsKey, result)) metricsCounter[tsKey] = int(result) p.delete(tsKey) p.execute() reportOpenTsdb(metricsCounter, opentsdbClient)
def import_time_series(ticker): df = web.DataReader(ticker, 'yahoo', start, end) metrics = potsdb.Client('localhost') for index, row in df.iterrows(): cob = int(time.mktime(index.date().timetuple()) * 1000) metrics.send('test.ts.equity.adjclose', row['Adj Close'], timestamp=cob, ticker=ticker) metrics.wait() metrics.close()
def getWriter(self): """ """ if self.metrics: self.counter = 0 self.stopWriter() while not self.metrics: try: self.metrics = potsdb.Client(self.ip, port=self.port, qsize=self.qsize, host_tag=self.host_tag, mps=self.mps, check_host=self.check_host) except socket.error as ex: print 'Received socket error %s' % str(ex) print 'Will sleep 15 seconds and try again' sleep(15)
def main(): try: #tsdb metrics = potsdb.Client(tsdbIp, port=tsdbPort,qsize=1000, host_tag=True, mps=100, check_host=True) #cpu vmstat = subprocess.check_output("vmstat -w 2 2",shell=True).split('\n'); #print vmstat vmstat_dict = vmstat_parse(vmstat) #print "\nCPU (metrics in %)" for k,v in vmstat_dict.iteritems(): metrics.send(k,v,host=hostname,instanceid=instanceid) print k,v,"host="+hostname,"instanceid"+instanceid #memory meminfo = subprocess.check_output("cat /proc/meminfo",shell=True).split('\n'); meminfo_dict = proc_meminfo_parse(meminfo) #system.mem.free,buffered(used for file buffers),cached(used as cache memory,),total,shared,usable(sum of free,buffered,cache),used,pct_usable(usable Ram as a fraction of total) memfree = meminfo_dict['MemFree']/(2**10) + meminfo_dict['Buffers']/2**10 + meminfo_dict['Cached']/2**10 memtotal = meminfo_dict['MemTotal']/2**10 memused = memtotal - memfree memutil = (memused*100)/memtotal swaptotal = meminfo_dict['SwapTotal']/(2**10) swapfree = meminfo_dict['SwapFree']/(2**10) mem_dict = {'system.mem.free':memfree,'system.mem.total':memtotal,'system.mem.used':memused,'system.mem.util':memutil,'system.mem.swap.total':swaptotal,'system.mem.swap.free':swapfree} #print "\nMemory (metrics in MB) \n" for k,v in mem_dict.iteritems(): metrics.send(k,v,host=hostname,instanceid=instanceid) print k,v,"host="+hostname,"instanceid"+instanceid #disk df = subprocess.check_output("df -m",shell=True).split('\n'); df_dict = df_parse(df) #print df_dict #print "\n Disk \n" for k,v in df_dict.iteritems(): metrics.send("system.disk.size",v['Size'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid) metrics.send("system.disk.used",v['Used'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid) metrics.send("system.disk.avail",v['Available'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid) metrics.send("system.disk.util",v['Use'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid) print "system.disk.size",v['Size'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid print "system.disk.used",v['Used'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid print "system.disk.avail",v['Available'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid print "system.disk.util",v['Use'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid cores = int(subprocess.check_output("nproc --all",shell=True)) metrics.send("system.cpu.cores",cores,host=hostname,instanceid=instanceid) print "system.cpu.cores",cores,"host="+hostname,"instanceid"+instanceid metrics.wait() #print "==cpu %(except cpu.numprocesswaiting int), disk MB(except disk.utill %) ,mem MB ,disk.blocks.read/write(blocks/s usually 1KB=1 block)=====" except Exception ,e : print "Exception:",e,"At Line Number {}".format(sys.exc_info()[-1].tb_lineno)
def init(self): """Init the connection to the OpenTSDB server.""" if not self.export_enable: return None try: db = potsdb.Client(self.host, port=int(self.port), check_host=True) except Exception as e: logger.critical("Cannot connect to OpenTSDB server %s:%s (%s)" % (self.host, self.port, e)) sys.exit(2) return db
def main(): interfaes_statictics_list_first = subprocess.check_output( "cat /proc/net/dev", shell=True).split('\n') first = proc_net_dev_parse(interfaes_statictics_list_first) subprocess.check_output("sleep 3", shell=True) interfaes_statictics_list_second = subprocess.check_output( "cat /proc/net/dev", shell=True).split('\n') second = proc_net_dev_parse(interfaes_statictics_list_second) difference(first, second) #transferrate() utilization() if_metrics = { 'rbytes': 'system.net.bytes_rcvd', 'tbytes': 'system.net.bytes_send', 'rutil': 'system.net.in_util', 'tutil': 'system.net.out_util', 'rrate': 'system.net.received_rate', 'trate': 'system.net.transmit_rate', 'rerrs': 'system.net.packets_in.error', 'terrs': 'system.net.packets_out.error', 'rdrop': 'system.net.packets_in.drops', 'tdrop': 'system.net.packets_out.drops' } #print if_metrics,if_metrics['rbytes'],if_metrics["rbytes"] try: #tsdb metrics = potsdb.Client(tsdbIp, port=tsdbPort, qsize=1000, host_tag=True, mps=100, check_host=True) for k, v in in_statictics.iteritems(): for k1, v1 in v.iteritems(): metrics.send(if_metrics[k1], v1, interface=k, host=hostname, instanceid=instanceid, scalinggroup=scaling_group) print if_metrics[ k1], v1, "interface=", k, "host=" + hostname, "instanceid" + instanceid, "scalinggroup" + scaling_group metrics.wait() print "========= drops packets,error int,util %,rate kB/s =======" except Exception, e: print "Exception:", e, "At Line Number {}".format( sys.exc_info()[-1].tb_lineno)
def __init__(self, settings=None, **kwargs): logger.debug("Configuring with settings: %s" % settings) self.settings = settings self.host = settings.get("TSDB_HOST", "defiant") self.port = settings.get("TSDB_PORT", 4242) self.qsize = settings.get("TSDB_QSIZE", 1000) self.mps = settings.get("TSDB_MPS", 100) self.host_tag = False self.check_host = True # connect self.client = potsdb.Client(self.host, port=self.port, qsize=self.qsize, host_tag=self.host_tag, mps=self.mps, check_host=self.check_host)
def main(config): tsdbIp = config['tsdb']['tsdbIp'] tsdbPort = config['tsdb']['tsdbPort'] hostname = config['node_config_files']['host'] interfaes_statictics_file = config['node_config_files']['network'] interfaes_statictics_list_first = open(interfaes_statictics_file, "r") first = proc_net_dev_parse(interfaes_statictics_list_first) print first time.sleep(3) interfaes_statictics_list_second = open(interfaes_statictics_file, "r") second = proc_net_dev_parse(interfaes_statictics_list_second) difference(first, second) transferrate() interface_f = config['node_config_files']['interface_speed'] utilization(interface_f) if_metrics = { 'rbytes': 'system.net.bytes_rcvd', 'tbytes': 'system.net.bytes_sent', 'rutil': 'system.net.in_util', 'tutil': 'system.net.out_util', 'rrate': 'system.net.received_rate', 'trate': 'system.net.transmit_rate', 'rerrs': 'system.net.packets_in.error', 'terrs': 'system.net.packets_out.error', 'rdrop': 'system.net.packets_in.drops', 'tdrop': 'system.net.packets_out.drops' } #print if_metrics,if_metrics['rbytes'],if_metrics["rbytes"] try: #tsdb metrics = potsdb.Client(tsdbIp, port=tsdbPort, qsize=1000, host_tag=True, mps=100, check_host=True) for k, v in in_statictics.iteritems(): for k1, v1 in v.iteritems(): metrics.send(if_metrics[k1], v1, interface=k, host=hostname) print if_metrics[k1], v1, "interface=", k, "host=", hostname metrics.wait() print "========= drops packets,error int,util %,rate kB/s =======" except Exception, e: print "Exception:", e, "At Line Number {}".format( sys.exc_info()[-1].tb_lineno)
def index() -> str: url = "http://opentsdb:4242" http_response = requests.get(url) print("Status code is: ") print(http_response.status_code) client = potsdb.Client('opentsdb', check_host=True, port=4242) print("Connection checked!\n") try: client.close() except Exception: traceback.print_exc() print("Client closed!\n") return "Ok"
def send_numeric_metric(metric_naming_list: list, metric_value, metric_timestamp: int) -> bool: """ Sends a single metric to the Time-Series database. :param metric_naming_list: The metric naming list. :param metric_value: The metric value in float, integer, or string (convertible to float or integer) format. :param metric_timestamp: The metric unix timestamp. :return: True if success, False otherwise. """ metric_name = format_metric_name(metric_naming_list) try: client = potsdb.Client(host=opentsdb_config['HOST'], port=opentsdb_config['PORT'], check_host=True) client.log(name=metric_name, val=metric_value, timestamp=metric_timestamp) client.close() return True except Exception as e: logger.error(e) return False
def __init__(self): config = configparser.ConfigParser() config.read('hive_to_opentsdb.conf') hive_host = config.get('hive', 'host') hive_port = config.get('hive', 'port') hive_db = config.get('hive', 'db') self.timestamp_column = config.get('hive', 'timestamp_column') self.tag_columns = config.get('hive', 'tag_columns').split(",") self.value_columns = config.get('hive', 'value_columns').split(",") opentsdb_host = config.get('opentsdb', 'host') opentsdb_port = config.get('opentsdb', 'port') self.table = config.get('hive', 'table') self.engine = create_engine('hive://{0}:{1}/{2}'.format( hive_host, hive_port, hive_db)) self.session = Session(bind=self.engine) self.metrics = potsdb.Client(opentsdb_host, port=opentsdb_port)
def pushMetrics(self, host=None, **kwargs): if not host: host = self.OPENTSDB_HOST port=self.OPENTSDB_PORT qsize=100000 host_tag=True mps=0 check_host=True for k in kwargs.keys(): if k == 'port': port = int(kwargs[k]) continue if k == 'qsize': qsize = int(kwargs[k]) continue if k == 'host_tag': host_tab = bool(kwargs[k]) continue if k == 'mps': mps = int(kwargs[k]) continue if k == 'check_host': check_host = bool(kwargs[k]) continue metrics = potsdb.Client(host, port=port, qsize=qsize, host_tag=host_tag, mps=mps, check_host=check_host) # send data to openTSDB for m,d in self.registry.iteritems(): for x in d: metrics.send(m, x.value, **x.tags) # wait until data are being sent out metrics.wait()
def main(config): try: tsdbIp = config['tsdb']['tsdbIp'] tsdbPort = config['tsdb']['tsdbPort'] hostname = config['node_config_files']['host'] #tsdb metrics = potsdb.Client(tsdbIp, port=tsdbPort, qsize=1000, host_tag=True, mps=100, check_host=True) #cpu f_c = cpu(config['node_config_files']['cpu_stats']) time.sleep(1) s_c = cpu(config['node_config_files']['cpu_stats']) cpu_r = {} for key in s_c: cpu_r[key] = s_c[key] - f_c[key] for key in cpu_r: k = "proc.cpu." + key metrics.send(k, cpu_r[key], host=hostname) print k, cpu_r[key], "host=", hostname print "proc.cpu.util ", (cpu_r['user'] + cpu_r['system']), "host=", hostname metrics.send("proc.cpu.util", (cpu_r['user'] + cpu_r['system']), host=hostname) loadavg = cpu_loadavg(config['node_config_files']['cpu_load']) for key in loadavg: k = "proc.cpu.load.avg." + key print k, loadavg[key], "host=" + hostname metrics.send(k, loadavg[key], host=hostname) #memory mem_file = config['node_config_files']['memory'] meminfo = open(mem_file, "r") meminfo_dict = proc_meminfo_parse(meminfo) systemmemfree = meminfo_dict['MemFree'] / (2**10) systemmembuffered = meminfo_dict['Buffers'] / 2**10 systemmemcached = meminfo_dict['Cached'] / 2**10 systemmemtotal = meminfo_dict['MemTotal'] / 2**10 systemmemshared = meminfo_dict['SwapCached'] / 2**10 systemmemused = (meminfo_dict['MemTotal'] - meminfo_dict['MemFree']) / 2**10 systemmemutil = (systemmemused * 100) / systemmemtotal mem_dict = { 'mem.free': systemmemfree, 'mem.buffered': systemmembuffered, 'mem.cached': systemmemcached, 'mem.total': systemmemtotal, 'mem.shared': systemmemshared, 'mem.used': systemmemused, 'mem.usage.percent': systemmemutil } for k, v in mem_dict.iteritems(): metrics.send(k, v, host=hostname) print k, v, "host=", hostname #disk f_d = diskstats_parse(["sda"], config['node_config_files']['disk']) time.sleep(1) s_d = diskstats_parse(["sda"], config['node_config_files']['disk']) for dev in s_d: reads = (s_d[dev]['reads'] - f_d[dev]['reads']) writes = (s_d[dev]['writes'] - f_d[dev]['writes']) rd_sectors = (s_d[dev]['rd_sectors'] - f_d[dev]['rd_sectors']) * 512 wr_sectors = (s_d[dev]['wr_sectors'] - f_d[dev]['wr_sectors']) * 512 metrics.send("system.io.reads", reads, device=s_d[dev]['dev'], host=hostname) metrics.send("system.io.writes", writes, device=s_d[dev]['dev'], host=hostname) metrics.send("system.io.rb_s", rd_sectors, device=s_d[dev]['dev'], host=hostname) metrics.send("system.io.wb_s", wr_sectors, device=s_d[dev]['dev'], host=hostname) print "disk.block.read", reads, "device=", s_d[dev][ 'dev'], "host=" + hostname #read requests issued to the device per second print "disk.block.write", writes, "device=", s_d[dev][ 'dev'], "host=" + hostname #write requests issued to the device per second print "system.io.rb_s", rd_sectors, "device=", s_d[dev][ 'dev'], "host=" + hostname # bytes reads to the device per second print "system.io.wb_s", wr_sectors, "device=", s_d[dev][ 'dev'], "host=" + hostname # bytes writes to the device per second #disk df = subprocess.check_output(['df', '-m']).split('\n') disk_root = config['node_config_files']['disk_root'] df_dict = df_parse(df, disk_root) for k, v in df_dict.iteritems(): metrics.send(k, v, host=hostname) print k, v, "host=", hostname, "directory=root" metrics.wait() print "========= cpu %(except cpu.numprocesswaiting int), disk MB(except disk.utill %) ,mem MB ,disk.blocks.read/write(blocks/s) =======" except Exception, e: print "Exception:", e, "At Line Number {}".format( sys.exc_info()[-1].tb_lineno)
def main(config): tsdbIp = config['tsdb']['tsdbIp'] tsdbPort = config['tsdb']['tsdbPort'] hostname = config['node_config_files']['host'] host_mem_size = round((float( read_file(config['node_config_files']['memory'], 1).split(":")[1].split()[0])) / (2**10), 3) string = config['docker_container_config_files']['string'] try: sleeptime = 3 a = get_all_values(string, config) time.sleep(sleeptime) b = get_all_values(string, config) #print a,b #cidname_dict = cidname_map() metrics = potsdb.Client(tsdbIp, port=tsdbPort, qsize=1000, host_tag=True, mps=100, check_host=True) for k, i in a.iteritems(): #memory max_mem = b[k]['max_memory'] / pow(2, 20) if max_mem > 22368547: max_mem = host_mem_size mem_usage = b[k]['usage_memory'] / pow(2, 20) docker_mem_percent = (mem_usage * 100) / max_mem mem_free = max_mem - mem_usage #cpu docker_cpu = ((b[k]['docker_cpu'] - a[k]['docker_cpu']) / sleeptime) * (pow(10, -9)) sys_cpu = ((b[k]['sys_cpu'] - a[k]['sys_cpu']) / sleeptime) * (pow( 10, -2)) docker_cpu_percent = (docker_cpu / sys_cpu) * 100 #disk docker_io_read = round( ((b[k]['docker_io_read'] - a[k]['docker_io_read']) / sleeptime) / pow(2, 20), 3) docker_io_write = round( ((b[k]['docker_io_write'] - a[k]['docker_io_write']) / sleeptime) / pow(2, 20), 3) #interface intface_stat1 = a[k]['intface_stat'] intface_stat2 = b[k]['intface_stat'] rx = round(((float(intface_stat2[1]) - float(intface_stat1[1])) / sleeptime) / pow(2, 10), 4) # - float(intface_stat1[1]) tx = round(((float(intface_stat2[9]) - float(intface_stat1[9])) / sleeptime) / pow(2, 10), 4) # - float(intface_stat1[9]) rx_drop = round( ((float(intface_stat2[4]) - float(intface_stat1[4])) / sleeptime)) # - float(intface_stat1[4]) tx_drop = round( ((float(intface_stat2[12]) - float(intface_stat1[12])) / sleeptime)) interface = intface_stat2[0].split(":")[0] #print k,docker_cpu_percent,docker_mem_percent,docker_io_read,docker_io_write,rx,tx,rx_drop,tx_drop metrics.send(metric6, rx, interface=interface, host=k, nodename=hostname) metrics.send(metric5, tx, interface=interface, host=k, nodename=hostname) metrics.send(metric12, rx_drop, interface=interface, host=k, nodename=hostname) metrics.send(metric13, tx_drop, interface=interface, host=k, nodename=hostname) print metric6, rx, "interface=" + interface, "host=" + k, "nodename=", hostname print metric5, tx, "interface=" + interface + "host=" + k, "nodename=" + hostname print metric12, rx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname print metric13, tx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname metrics.send(metric1, round(docker_cpu_percent, 3), host=k, nodename=hostname) print metric1, round(docker_cpu_percent, 3), "host=" + k, "nodename=" + hostname metrics.send(metric2, mem_usage, host=k, nodename=hostname) print metric2, mem_usage, "host=" + k, "nodename=" + hostname metrics.send(metric9, mem_free, host=k, nodename=hostname) print metric9, mem_free, "host=" + k, "nodename=" + hostname metrics.send(metric3, max_mem, host=k, nodename=hostname) print metric3, max_mem, "host=" + k, "nodename=" + hostname metrics.send(metric4, round(docker_mem_percent, 3), host=k, nodename=hostname) print metric4, round(docker_mem_percent, 3), "host=" + k, "nodename=" + hostname metrics.send(metric7, docker_io_read, host=k, nodename=hostname) print metric7, docker_io_read, "host=" + k, "nodename=" + hostname metrics.send(metric8, docker_io_write, host=k, nodename=hostname) print metric8, docker_io_write, "host=" + k, "nodename=" + hostname print "=======================================================================================================" except Exception, e: print "Exception:", e, "At Line Number {}".format( sys.exc_info()[-1].tb_lineno)
'net.in_packet.error': in_packet_error_eno2, 'net.out_packet.error': out_packet_error_eno2, 'net.qlen': metric_value_2.get('ifOutQLen_eno2'), } metric_server_t1 = { 'net.tcp.in_segment.error': in_tcp_seg_error, 'net.tcp.out_segment.reset': out_tcp_resets, 'net.retrans': percent_retrans, 'net.in_ip_fragment': in_frag_percent, 'net.in_out_fragment': out_frag_percent } metrics = potsdb.Client('175.126.104.46', port=4343, qsize=1000, host_tag=True, mps=100, check_host=True) #server related metrics push def search(values, searchFor): for k in values: if searchFor in k: #print k,values.get(k),"\n" # return k,values.get(k) list_1 = k.split("_") #print "list is",list_1[0],list_1[1] list_1[0] = "net." + list_1[0] print "%s %s %s" % (list_1[0], values.get(k), hostname) #metrics.send(list_1[0],values.get(k),server_ip=list_1[1])
# - logging configuration logging.basicConfig() logger = logging.getLogger('netclient') logger.setLevel(logging.DEBUG) host_ip = "" host_port = 9033 #random mode = "stream" nums = 0 db_host = 'localhost' db_port = 4242 metrics = potsdb.Client(db_host, port=db_port, qsize=25, mps=20) badsite_keywords = { "facebook", "twitter", "reddit", "netflix", "fb", "messenger" } path = "" def encodePath(path): pathStrs = "" for pe in path: pstr = pe.name if pe.key: for k, v in pe.key.iteritems(): pstr += "[" + str(k) + "=" + str(v) + "]" pathStrs = pathStrs + "." + pstr
for element in countdict: countdict[element] = len(countdict[element]) topk_keys = sorted(countdict, key = countdict.get, reverse = True)[:kDomainsToStore] for key in topk_keys: for value in set(dict[key]): #(Metric name, value, tags ...) metrics.send('Domains', dict[key].count(value), timestamp= item["timeStamp"], domain = key, serverId = item["serverId"]) """ Main """ if __name__ == "__main__": metrics = potsdb.Client(config["openTSDB"]["address"], config["openTSDB"]["port"]) answersPerSecond = AnswerPerSecondListener(r, ['AnswersPerSecond']) answersPerSecond.start() queriesPerSecond = QueriesPerSecondListener(r, ['QueriesPerSecond']) queriesPerSecond.start() queriesSummary = QueriesSummaryListener(r, ['QueriesSummary']) queriesSummary.start() topK = TopKListener(r, ['TopK']) ##topK.start() #Not used channel topKWithIP = TopKWithIPListener(r, ['TopKWithIP']) topKWithIP.start()
def execute(configIn=None, loggerIn=None): """ Execute main script for DTN-RM Agent output preparation """ config, logger = getDefaultConfigAgent(COMPONENT, configIn, loggerIn) if not config.getboolean(COMPONENT, 'enabled'): logger.info('Component is not enabled in configuration.') return success, outConfig = getNetdataConfig() if not success: logger.warning('Failure receiving netdata configuration file. Exiting') exit(1) workDir = config.get('agent', 'PRIVATE_DIR') + "/DTNRM/" agentDB = contentDB(logger=logger, config=config) # TODO Implement enabled option and debug per each component. # if not config.getboolean(COMPONENT, 'enabled'): # return # http://vocms025.cern.ch:19999/api/v1/alarms?active&_=1508422746160 agents = makeGETRequest("http://localhost:19999/api/v1/alarms?active") out = { 'netdata.warnings': 0, 'netdata.critical': 0, 'sense.nodes': 1, 'sense.vlansInUse': 0, 'sense.vlansFree': 0, 'sense.vlansTotal': 0, 'sense.agentError': 0, 'sense.agentWarning': 0, 'sense.agent.recurringactions.status': 0, 'sense.agent.recurringactions.runtime': 0, 'sense.agent.ruler.status': 0, 'sense.agent.ruler.runtime': 0 } for _dKey, dVals in agents[1]['alarms'].items(): if dVals['status'] == 'WARNING': out['netdata.warnings'] += 1 elif dVals['status'] == 'CRITICAL': out['netdata.critical'] += 1 agentOut = agentDB.getFileContentAsJson(workDir + "/latest-out.json") for _interf, interfDict in agentOut['NetInfo'].items(): if 'vlan_range' in interfDict: lowR, highR = interfDict['vlan_range'].split(',') total = int(highR) - int(lowR) if total < 0: # TODO print error out['sense.agentError'] += 1 else: out['sense.vlansTotal'] += total out['sense.vlansInUse'] = len(interfDict['vlans']) free = out['sense.vlansTotal'] - out['sense.vlansInUse'] if free < 0: # TODO: print error out['sense.agentError'] += 1 else: out['sense.vlansFree'] = free if 'destination' in outConfig: splVal = outConfig['destination'].split(':') if len(splVal) == 3: destHostname = splVal[1] destPort = splVal[2] elif len(splVal) == 2: destHostname = splVal[0] destPort = splVal[1] else: print 'FAILURE. Was expecting protocol:ip:port or ip:port... Got Value: %s' % outConfig[ 'destination'] return else: print 'FAILURE. Backend is not configured' return metrics = potsdb.Client(destHostname, port=destPort, qsize=1000, host_tag=True, mps=100, check_host=True) for key, value in out.items(): metrics.send("monit.%s" % key, value, **outConfig['tags']) # waits for all outstanding metrics to be sent and background thread closes metrics.wait()
#!/usr/bin/env python ''' Author: Veerndra.K Description: A simple example snippet, sends metrics to OpenTSDB Requried Modules: potsdb (pip install potsdb) ''' import potsdb import time tsdbIp = "127.0.0.1" tsdbPort = 4343 interval = 5 try: metrics = potsdb.Client(tsdbIp, tsdbPort, qsize=1000, host_tag=True, mps=100, check_host=True) ts = int(time.time()) metrics.send("sample.test", 1, host="my-host") print "sample.test 1 host=my-host" metrics.wait() time.sleep(1) except: pass time.sleep(5)
def metrics_send(n, tn, v): name = "%s.%s.%s.%s" % (NAME_PREFIX, SPATH, tn, n) print "sending metric %s = %d " % (name, v) metrics = potsdb.Client(METRICS_HOST) metrics.send(name, v) metrics.wait()
#!/usr/bin/python import potsdb import sys # set this to the hostname of the host running opentsdb METRICS_HOST = 'node71' if (len(sys.argv) != 3): print "usage: %s %s %s" (sys.argv[0], "<metric_name>", "<metric_value>") sys.exit(-1) MNAME = sys.argv[1] MVALUE = int(sys.argv[2]) print "sending metric %s = %d " % (MNAME, MVALUE) metrics = potsdb.Client(METRICS_HOST) metrics.send(MNAME, MVALUE) metrics.wait() print "done" sys.exit(0)
# current datetime current_datetime = datetime.datetime.utcnow() for key in api_keys: try: # get server list https_connection = http.client.HTTPSConnection("api.leaseweb.com") https_headers = { 'x-lsw-auth': api_keys[key]} https_connection.request("GET", "/bareMetals/v2/servers", headers=https_headers) https_response = https_connection.getresponse() https_data = https_response.read() json_servers = json.loads(https_data.decode("utf-8")) # connect to opentsdb metrics = potsdb.Client(opentsdb_server, port=opentsdb_port,check_host=False) except: continue if not 'servers' in json_servers: print('LeaseWeb', key, 'returned no data') continue # for each server for bareMetal in json_servers['servers']: try: # get id and name bareMetalId = bareMetal['id'] serverName = bareMetal['contract']['internalReference'] # get datatraffic
def main(): host_mem_size = round( (float(read_file(file_locations['memory'], 1, 1))) / (2**10), 3) try: sleeptime = 3 a = get_all_values() time.sleep(sleeptime) b = get_all_values() metrics = potsdb.Client(tsdbIp, port=tsdbPort, qsize=1000, host_tag=True, mps=100, check_host=True) for k, i in a.iteritems(): #memory max_mem = b[k]['max_memory'] / pow(2, 20) if max_mem > 22368547: max_mem = host_mem_size mem_usage = b[k]['usage_memory'] / pow(2, 20) docker_mem_percent = (mem_usage * 100) / max_mem mem_free = max_mem - mem_usage #cpu docker_cpu = ((b[k]['docker_cpu'] - a[k]['docker_cpu']) / sleeptime) * (pow(10, -9)) sys_cpu = ((b[k]['sys_cpu'] - a[k]['sys_cpu']) / sleeptime) * (pow( 10, -2)) docker_cpu_percent = (docker_cpu / sys_cpu) * 100 #disk docker_io_read = round( ((b[k]['docker_io_read'] - a[k]['docker_io_read']) / sleeptime) / pow(2, 20), 3) docker_io_write = round( ((b[k]['docker_io_write'] - a[k]['docker_io_write']) / sleeptime) / pow(2, 20), 3) #interface intface_stat1 = a[k]['intface_stat'] intface_stat2 = b[k]['intface_stat'] rx = round(((float(intface_stat2[1]) - float(intface_stat1[1])) / sleeptime) / pow(2, 10), 4) # - float(intface_stat1[1]) tx = round(((float(intface_stat2[9]) - float(intface_stat1[9])) / sleeptime) / pow(2, 10), 4) # - float(intface_stat1[9]) rx_drop = round( ((float(intface_stat2[4]) - float(intface_stat1[4])) / sleeptime)) # - float(intface_stat1[4]) tx_drop = round( ((float(intface_stat2[12]) - float(intface_stat1[12])) / sleeptime)) interface = intface_stat2[0].split(":")[0] metrics.send(metric6, rx, interface=interface, host=k, nodename=hostname) metrics.send(metric5, tx, interface=interface, host=k, nodename=hostname) metrics.send(metric12, rx_drop, interface=interface, host=k, nodename=hostname) metrics.send(metric13, tx_drop, interface=interface, host=k, nodename=hostname) print metric6, rx, "interface=" + interface, "host=" + k, "nodename=", hostname print metric5, tx, "interface=" + interface + "host=" + k, "nodename=" + hostname print metric12, rx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname print metric13, tx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname metrics.send(metric1, round(docker_cpu_percent, 3), host=k, nodename=hostname) print metric1, round(docker_cpu_percent, 3), "host=" + k, "nodename=" + hostname metrics.send(metric2, mem_usage, host=k, nodename=hostname) print metric2, mem_usage, "host=" + k, "nodename=" + hostname metrics.send(metric9, mem_free, host=k, nodename=hostname) print metric9, mem_free, "host=" + k, "nodename=" + hostname metrics.send(metric3, max_mem, host=k, nodename=hostname) print metric3, max_mem, "host=" + k, "nodename=" + hostname metrics.send(metric4, round(docker_mem_percent, 3), host=k, nodename=hostname) print metric4, round(docker_mem_percent, 3), "host=" + k, "nodename=" + hostname metrics.send(metric7, docker_io_read, host=k, nodename=hostname) print metric7, docker_io_read, "host=" + k, "nodename=" + hostname metrics.send(metric8, docker_io_write, host=k, nodename=hostname) print metric8, docker_io_write, "host=" + k, "nodename=" + hostname print "=======================================================================================================" except: print "=============TRACKBACK=============" print traceback.format_exc()
def __init__(self): self.client = potsdb.Client('localhost')
# - logging configuration logging.basicConfig() logger = logging.getLogger('test-client') logger.setLevel(logging.DEBUG) host_ip = "localhost" host_port = 80050 mode = "stream" db_host = '127.0.0.1' #db_port = 8086 db_port = 4242 metrics = potsdb.Client(db_host, port=db_port) #client = InfluxDBClient('localhost', 8086, 'root', 'root', 'example') #client.create_database('example') def encodePath(path): pathStrs = "" for pe in path: pstr = pe.name if pe.key: for k, v in pe.key.iteritems(): pstr += "[" + str(k) + "=" + str(v) + "]" pathStrs = pathStrs + "." + pstr return pathStrs[1:] def mapToJson(k,v,data):
config.openTSDB['host'] = config.openTSDB['host'] if os.getenv( 'OPENTSDB_HOST', 'None') == 'None' else os.getenv('OPENTSDB_HOST', '127.0.0.1') config.openTSDB['port'] = config.openTSDB['port'] if os.getenv( 'OPENTSDB_PORT', 'None') == 'None' else int( os.getenv('OPENTSDB_PORT', '4242')) config.GlobalAuthData[ 'username'] = config.GlobalAuthData['username'] if os.getenv( 'AUTH_USERNAME', 'None') == 'None' else os.getenv('AUTH_USERNAME', '') config.GlobalAuthData[ 'password'] = config.GlobalAuthData['password'] if os.getenv( 'AUTH_PASSWORD', 'None') == 'None' else os.getenv('AUTH_PASSWORD', '') metrics = potsdb.Client(config.openTSDB['host'], port=config.openTSDB['port'], check_host=True) now_last = datetime.now().date() utc_date = None if '-' in num_env: utc_date = datetime( *(time.strptime(num_env, '%Y-%m-%d')[0:6])) - timedelta(days=1) else: utc_date = datetime.now() - timedelta(days=int(num_env) + 1) jqls = [{ "new": config.Jira['jql_new'] }, { "closed": config.Jira['jql_closed']
def __init__(self, *args, **kwargs): super(RyuToOpentsdb, self).__init__(*args, **kwargs) self.datapaths = {} self.collector_thread = hub.spawn(self.run_stats_collector) self.metrics = potsdb.Client(os.getenv('OTSDB_HOST', 'opentsdb')) self.metric_prefix = os.getenv('OTSDB_METRIC_PREFIX', 'oftester')
def _get_client(**kwargs): my_kwargs = {"port": PORT, "check_host": False, "test_mode": True} my_kwargs.update(kwargs) return potsdb.Client(HOST, **my_kwargs)