def reportMetrics(pool):
p = redis.StrictRedis(connection_pool=pool).pipeline(transaction=False)
opentsdbClient = potsdb.Client(OPENTSDB_URL)
while True:
time.sleep(REPORT_INTERVAL * 60)
lastWindow = getTimestampByMinute(datetime.utcnow() -
timedelta(minutes=REPORT_DELAY))
p.zrangebyscore('TS', 0, lastWindow)
p.zremrangebyscore('TS', 0, lastWindow)
tsRange = p.execute()
metricsCounter = {}
for tsKey in tsRange[0]:
p.hgetall(tsKey)
results = p.execute()
for tsKey, result in zip(tsRange[0], results):
metricsCounter[tsKey] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
for key, value in result.iteritems():
index = COUNTER_KEYS[key]
metricsCounter[tsKey][index] = int(value)
p.hdel(tsKey, key)
p.execute()
reportOpenTsdb(metricsCounter, opentsdbClient)
def get_db_client(self):
return potsdb.Client(host=settings.OPENTSDB_HOST,
port=settings.OPENTSDB_PORT,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
def reportMetrics(pool):
p = redis.StrictRedis(connection_pool=pool).pipeline(transaction=False)
opentsdbClient = potsdb.Client(OPENTSDB_URL)
while True:
time.sleep(REPORT_INTERVAL * 60)
lastWindow = getTimestampByMinute(datetime.utcnow() -
timedelta(minutes=REPORT_DELAY))
p.zrangebyscore('TS', 0, lastWindow)
p.zremrangebyscore('TS', 0, lastWindow)
tsRange = p.execute()
metricsCounter = {}
for tsKey in tsRange[0]:
p.get(tsKey)
results = p.execute()
for tsKey, result in zip(tsRange[0], results):
#logger.info("tsKey:%s, result:%s" % (tsKey, result))
metricsCounter[tsKey] = int(result)
p.delete(tsKey)
p.execute()
reportOpenTsdb(metricsCounter, opentsdbClient)
def import_time_series(ticker):
df = web.DataReader(ticker, 'yahoo', start, end)
metrics = potsdb.Client('localhost')
for index, row in df.iterrows():
cob = int(time.mktime(index.date().timetuple()) * 1000)
metrics.send('test.ts.equity.adjclose', row['Adj Close'], timestamp=cob, ticker=ticker)
metrics.wait()
metrics.close()
def getWriter(self):
""" """
if self.metrics:
self.counter = 0
self.stopWriter()
while not self.metrics:
try:
self.metrics = potsdb.Client(self.ip, port=self.port, qsize=self.qsize, host_tag=self.host_tag, mps=self.mps, check_host=self.check_host)
except socket.error as ex:
print 'Received socket error %s' % str(ex)
print 'Will sleep 15 seconds and try again'
sleep(15)
def main():
try:
#tsdb
metrics = potsdb.Client(tsdbIp, port=tsdbPort,qsize=1000, host_tag=True, mps=100, check_host=True)
#cpu
vmstat = subprocess.check_output("vmstat -w 2 2",shell=True).split('\n');
#print vmstat
vmstat_dict = vmstat_parse(vmstat)
#print "\nCPU (metrics in %)"
for k,v in vmstat_dict.iteritems():
metrics.send(k,v,host=hostname,instanceid=instanceid)
print k,v,"host="+hostname,"instanceid"+instanceid
#memory
meminfo = subprocess.check_output("cat /proc/meminfo",shell=True).split('\n');
meminfo_dict = proc_meminfo_parse(meminfo)
#system.mem.free,buffered(used for file buffers),cached(used as cache memory,),total,shared,usable(sum of free,buffered,cache),used,pct_usable(usable Ram as a fraction of total)
memfree = meminfo_dict['MemFree']/(2**10) + meminfo_dict['Buffers']/2**10 + meminfo_dict['Cached']/2**10
memtotal = meminfo_dict['MemTotal']/2**10
memused = memtotal - memfree
memutil = (memused*100)/memtotal
swaptotal = meminfo_dict['SwapTotal']/(2**10)
swapfree = meminfo_dict['SwapFree']/(2**10)
mem_dict = {'system.mem.free':memfree,'system.mem.total':memtotal,'system.mem.used':memused,'system.mem.util':memutil,'system.mem.swap.total':swaptotal,'system.mem.swap.free':swapfree}
#print "\nMemory (metrics in MB) \n"
for k,v in mem_dict.iteritems():
metrics.send(k,v,host=hostname,instanceid=instanceid)
print k,v,"host="+hostname,"instanceid"+instanceid
#disk
df = subprocess.check_output("df -m",shell=True).split('\n');
df_dict = df_parse(df)
#print df_dict
#print "\n Disk \n"
for k,v in df_dict.iteritems():
metrics.send("system.disk.size",v['Size'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid)
metrics.send("system.disk.used",v['Used'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid)
metrics.send("system.disk.avail",v['Available'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid)
metrics.send("system.disk.util",v['Use'],host=hostname,device=k,mounton=v['Mountedon'],instanceid=instanceid)
print "system.disk.size",v['Size'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid
print "system.disk.used",v['Used'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid
print "system.disk.avail",v['Available'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid
print "system.disk.util",v['Use'],"host="+hostname,"device="+k,"mounton="+v['Mountedon'],"instanceid"+instanceid
cores = int(subprocess.check_output("nproc --all",shell=True))
metrics.send("system.cpu.cores",cores,host=hostname,instanceid=instanceid)
print "system.cpu.cores",cores,"host="+hostname,"instanceid"+instanceid
metrics.wait()
#print "==cpu %(except cpu.numprocesswaiting int), disk MB(except disk.utill %) ,mem MB ,disk.blocks.read/write(blocks/s usually 1KB=1 block)====="
except Exception ,e :
print "Exception:",e,"At Line Number {}".format(sys.exc_info()[-1].tb_lineno)
def init(self):
"""Init the connection to the OpenTSDB server."""
if not self.export_enable:
return None
try:
db = potsdb.Client(self.host, port=int(self.port), check_host=True)
except Exception as e:
logger.critical("Cannot connect to OpenTSDB server %s:%s (%s)" %
(self.host, self.port, e))
sys.exit(2)
return db
def main():
interfaes_statictics_list_first = subprocess.check_output(
"cat /proc/net/dev", shell=True).split('\n')
first = proc_net_dev_parse(interfaes_statictics_list_first)
subprocess.check_output("sleep 3", shell=True)
interfaes_statictics_list_second = subprocess.check_output(
"cat /proc/net/dev", shell=True).split('\n')
second = proc_net_dev_parse(interfaes_statictics_list_second)
difference(first, second)
#transferrate()
utilization()
if_metrics = {
'rbytes': 'system.net.bytes_rcvd',
'tbytes': 'system.net.bytes_send',
'rutil': 'system.net.in_util',
'tutil': 'system.net.out_util',
'rrate': 'system.net.received_rate',
'trate': 'system.net.transmit_rate',
'rerrs': 'system.net.packets_in.error',
'terrs': 'system.net.packets_out.error',
'rdrop': 'system.net.packets_in.drops',
'tdrop': 'system.net.packets_out.drops'
}
#print if_metrics,if_metrics['rbytes'],if_metrics["rbytes"]
try:
#tsdb
metrics = potsdb.Client(tsdbIp,
port=tsdbPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
for k, v in in_statictics.iteritems():
for k1, v1 in v.iteritems():
metrics.send(if_metrics[k1],
v1,
interface=k,
host=hostname,
instanceid=instanceid,
scalinggroup=scaling_group)
print if_metrics[
k1], v1, "interface=", k, "host=" + hostname, "instanceid" + instanceid, "scalinggroup" + scaling_group
metrics.wait()
print "========= drops packets,error int,util %,rate kB/s ======="
except Exception, e:
print "Exception:", e, "At Line Number {}".format(
sys.exc_info()[-1].tb_lineno)
def __init__(self, settings=None, **kwargs):
logger.debug("Configuring with settings: %s" % settings)
self.settings = settings
self.host = settings.get("TSDB_HOST", "defiant")
self.port = settings.get("TSDB_PORT", 4242)
self.qsize = settings.get("TSDB_QSIZE", 1000)
self.mps = settings.get("TSDB_MPS", 100)
self.host_tag = False
self.check_host = True
# connect
self.client = potsdb.Client(self.host,
port=self.port,
qsize=self.qsize,
host_tag=self.host_tag,
mps=self.mps,
check_host=self.check_host)
def main(config):
tsdbIp = config['tsdb']['tsdbIp']
tsdbPort = config['tsdb']['tsdbPort']
hostname = config['node_config_files']['host']
interfaes_statictics_file = config['node_config_files']['network']
interfaes_statictics_list_first = open(interfaes_statictics_file, "r")
first = proc_net_dev_parse(interfaes_statictics_list_first)
print first
time.sleep(3)
interfaes_statictics_list_second = open(interfaes_statictics_file, "r")
second = proc_net_dev_parse(interfaes_statictics_list_second)
difference(first, second)
transferrate()
interface_f = config['node_config_files']['interface_speed']
utilization(interface_f)
if_metrics = {
'rbytes': 'system.net.bytes_rcvd',
'tbytes': 'system.net.bytes_sent',
'rutil': 'system.net.in_util',
'tutil': 'system.net.out_util',
'rrate': 'system.net.received_rate',
'trate': 'system.net.transmit_rate',
'rerrs': 'system.net.packets_in.error',
'terrs': 'system.net.packets_out.error',
'rdrop': 'system.net.packets_in.drops',
'tdrop': 'system.net.packets_out.drops'
}
#print if_metrics,if_metrics['rbytes'],if_metrics["rbytes"]
try:
#tsdb
metrics = potsdb.Client(tsdbIp,
port=tsdbPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
for k, v in in_statictics.iteritems():
for k1, v1 in v.iteritems():
metrics.send(if_metrics[k1], v1, interface=k, host=hostname)
print if_metrics[k1], v1, "interface=", k, "host=", hostname
metrics.wait()
print "========= drops packets,error int,util %,rate kB/s ======="
except Exception, e:
print "Exception:", e, "At Line Number {}".format(
sys.exc_info()[-1].tb_lineno)
def index() -> str:
url = "http://opentsdb:4242"
http_response = requests.get(url)
print("Status code is: ")
print(http_response.status_code)
client = potsdb.Client('opentsdb', check_host=True, port=4242)
print("Connection checked!\n")
try:
client.close()
except Exception:
traceback.print_exc()
print("Client closed!\n")
return "Ok"
def send_numeric_metric(metric_naming_list: list, metric_value, metric_timestamp: int) -> bool:
"""
Sends a single metric to the Time-Series database.
:param metric_naming_list: The metric naming list.
:param metric_value: The metric value in float, integer, or string (convertible to float or integer) format.
:param metric_timestamp: The metric unix timestamp.
:return: True if success, False otherwise.
"""
metric_name = format_metric_name(metric_naming_list)
try:
client = potsdb.Client(host=opentsdb_config['HOST'], port=opentsdb_config['PORT'], check_host=True)
client.log(name=metric_name, val=metric_value, timestamp=metric_timestamp)
client.close()
return True
except Exception as e:
logger.error(e)
return False
def __init__(self):
config = configparser.ConfigParser()
config.read('hive_to_opentsdb.conf')
hive_host = config.get('hive', 'host')
hive_port = config.get('hive', 'port')
hive_db = config.get('hive', 'db')
self.timestamp_column = config.get('hive', 'timestamp_column')
self.tag_columns = config.get('hive', 'tag_columns').split(",")
self.value_columns = config.get('hive', 'value_columns').split(",")
opentsdb_host = config.get('opentsdb', 'host')
opentsdb_port = config.get('opentsdb', 'port')
self.table = config.get('hive', 'table')
self.engine = create_engine('hive://{0}:{1}/{2}'.format(
hive_host, hive_port, hive_db))
self.session = Session(bind=self.engine)
self.metrics = potsdb.Client(opentsdb_host, port=opentsdb_port)
def pushMetrics(self, host=None, **kwargs):
if not host:
host = self.OPENTSDB_HOST
port=self.OPENTSDB_PORT
qsize=100000
host_tag=True
mps=0
check_host=True
for k in kwargs.keys():
if k == 'port':
port = int(kwargs[k])
continue
if k == 'qsize':
qsize = int(kwargs[k])
continue
if k == 'host_tag':
host_tab = bool(kwargs[k])
continue
if k == 'mps':
mps = int(kwargs[k])
continue
if k == 'check_host':
check_host = bool(kwargs[k])
continue
metrics = potsdb.Client(host, port=port, qsize=qsize, host_tag=host_tag, mps=mps, check_host=check_host)
# send data to openTSDB
for m,d in self.registry.iteritems():
for x in d:
metrics.send(m, x.value, **x.tags)
# wait until data are being sent out
metrics.wait()
def main(config):
try:
tsdbIp = config['tsdb']['tsdbIp']
tsdbPort = config['tsdb']['tsdbPort']
hostname = config['node_config_files']['host']
#tsdb
metrics = potsdb.Client(tsdbIp,
port=tsdbPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
#cpu
f_c = cpu(config['node_config_files']['cpu_stats'])
time.sleep(1)
s_c = cpu(config['node_config_files']['cpu_stats'])
cpu_r = {}
for key in s_c:
cpu_r[key] = s_c[key] - f_c[key]
for key in cpu_r:
k = "proc.cpu." + key
metrics.send(k, cpu_r[key], host=hostname)
print k, cpu_r[key], "host=", hostname
print "proc.cpu.util ", (cpu_r['user'] +
cpu_r['system']), "host=", hostname
metrics.send("proc.cpu.util", (cpu_r['user'] + cpu_r['system']),
host=hostname)
loadavg = cpu_loadavg(config['node_config_files']['cpu_load'])
for key in loadavg:
k = "proc.cpu.load.avg." + key
print k, loadavg[key], "host=" + hostname
metrics.send(k, loadavg[key], host=hostname)
#memory
mem_file = config['node_config_files']['memory']
meminfo = open(mem_file, "r")
meminfo_dict = proc_meminfo_parse(meminfo)
systemmemfree = meminfo_dict['MemFree'] / (2**10)
systemmembuffered = meminfo_dict['Buffers'] / 2**10
systemmemcached = meminfo_dict['Cached'] / 2**10
systemmemtotal = meminfo_dict['MemTotal'] / 2**10
systemmemshared = meminfo_dict['SwapCached'] / 2**10
systemmemused = (meminfo_dict['MemTotal'] -
meminfo_dict['MemFree']) / 2**10
systemmemutil = (systemmemused * 100) / systemmemtotal
mem_dict = {
'mem.free': systemmemfree,
'mem.buffered': systemmembuffered,
'mem.cached': systemmemcached,
'mem.total': systemmemtotal,
'mem.shared': systemmemshared,
'mem.used': systemmemused,
'mem.usage.percent': systemmemutil
}
for k, v in mem_dict.iteritems():
metrics.send(k, v, host=hostname)
print k, v, "host=", hostname
#disk
f_d = diskstats_parse(["sda"], config['node_config_files']['disk'])
time.sleep(1)
s_d = diskstats_parse(["sda"], config['node_config_files']['disk'])
for dev in s_d:
reads = (s_d[dev]['reads'] - f_d[dev]['reads'])
writes = (s_d[dev]['writes'] - f_d[dev]['writes'])
rd_sectors = (s_d[dev]['rd_sectors'] -
f_d[dev]['rd_sectors']) * 512
wr_sectors = (s_d[dev]['wr_sectors'] -
f_d[dev]['wr_sectors']) * 512
metrics.send("system.io.reads",
reads,
device=s_d[dev]['dev'],
host=hostname)
metrics.send("system.io.writes",
writes,
device=s_d[dev]['dev'],
host=hostname)
metrics.send("system.io.rb_s",
rd_sectors,
device=s_d[dev]['dev'],
host=hostname)
metrics.send("system.io.wb_s",
wr_sectors,
device=s_d[dev]['dev'],
host=hostname)
print "disk.block.read", reads, "device=", s_d[dev][
'dev'], "host=" + hostname #read requests issued to the device per second
print "disk.block.write", writes, "device=", s_d[dev][
'dev'], "host=" + hostname #write requests issued to the device per second
print "system.io.rb_s", rd_sectors, "device=", s_d[dev][
'dev'], "host=" + hostname # bytes reads to the device per second
print "system.io.wb_s", wr_sectors, "device=", s_d[dev][
'dev'], "host=" + hostname # bytes writes to the device per second
#disk
df = subprocess.check_output(['df', '-m']).split('\n')
disk_root = config['node_config_files']['disk_root']
df_dict = df_parse(df, disk_root)
for k, v in df_dict.iteritems():
metrics.send(k, v, host=hostname)
print k, v, "host=", hostname, "directory=root"
metrics.wait()
print "========= cpu %(except cpu.numprocesswaiting int), disk MB(except disk.utill %) ,mem MB ,disk.blocks.read/write(blocks/s) ======="
except Exception, e:
print "Exception:", e, "At Line Number {}".format(
sys.exc_info()[-1].tb_lineno)
def main(config):
tsdbIp = config['tsdb']['tsdbIp']
tsdbPort = config['tsdb']['tsdbPort']
hostname = config['node_config_files']['host']
host_mem_size = round((float(
read_file(config['node_config_files']['memory'],
1).split(":")[1].split()[0])) / (2**10), 3)
string = config['docker_container_config_files']['string']
try:
sleeptime = 3
a = get_all_values(string, config)
time.sleep(sleeptime)
b = get_all_values(string, config)
#print a,b
#cidname_dict = cidname_map()
metrics = potsdb.Client(tsdbIp,
port=tsdbPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
for k, i in a.iteritems():
#memory
max_mem = b[k]['max_memory'] / pow(2, 20)
if max_mem > 22368547:
max_mem = host_mem_size
mem_usage = b[k]['usage_memory'] / pow(2, 20)
docker_mem_percent = (mem_usage * 100) / max_mem
mem_free = max_mem - mem_usage
#cpu
docker_cpu = ((b[k]['docker_cpu'] - a[k]['docker_cpu']) /
sleeptime) * (pow(10, -9))
sys_cpu = ((b[k]['sys_cpu'] - a[k]['sys_cpu']) / sleeptime) * (pow(
10, -2))
docker_cpu_percent = (docker_cpu / sys_cpu) * 100
#disk
docker_io_read = round(
((b[k]['docker_io_read'] - a[k]['docker_io_read']) / sleeptime)
/ pow(2, 20), 3)
docker_io_write = round(
((b[k]['docker_io_write'] - a[k]['docker_io_write']) /
sleeptime) / pow(2, 20), 3)
#interface
intface_stat1 = a[k]['intface_stat']
intface_stat2 = b[k]['intface_stat']
rx = round(((float(intface_stat2[1]) - float(intface_stat1[1])) /
sleeptime) / pow(2, 10),
4) # - float(intface_stat1[1])
tx = round(((float(intface_stat2[9]) - float(intface_stat1[9])) /
sleeptime) / pow(2, 10),
4) # - float(intface_stat1[9])
rx_drop = round(
((float(intface_stat2[4]) - float(intface_stat1[4])) /
sleeptime)) # - float(intface_stat1[4])
tx_drop = round(
((float(intface_stat2[12]) - float(intface_stat1[12])) /
sleeptime))
interface = intface_stat2[0].split(":")[0]
#print k,docker_cpu_percent,docker_mem_percent,docker_io_read,docker_io_write,rx,tx,rx_drop,tx_drop
metrics.send(metric6,
rx,
interface=interface,
host=k,
nodename=hostname)
metrics.send(metric5,
tx,
interface=interface,
host=k,
nodename=hostname)
metrics.send(metric12,
rx_drop,
interface=interface,
host=k,
nodename=hostname)
metrics.send(metric13,
tx_drop,
interface=interface,
host=k,
nodename=hostname)
print metric6, rx, "interface=" + interface, "host=" + k, "nodename=", hostname
print metric5, tx, "interface=" + interface + "host=" + k, "nodename=" + hostname
print metric12, rx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname
print metric13, tx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname
metrics.send(metric1,
round(docker_cpu_percent, 3),
host=k,
nodename=hostname)
print metric1, round(docker_cpu_percent,
3), "host=" + k, "nodename=" + hostname
metrics.send(metric2, mem_usage, host=k, nodename=hostname)
print metric2, mem_usage, "host=" + k, "nodename=" + hostname
metrics.send(metric9, mem_free, host=k, nodename=hostname)
print metric9, mem_free, "host=" + k, "nodename=" + hostname
metrics.send(metric3, max_mem, host=k, nodename=hostname)
print metric3, max_mem, "host=" + k, "nodename=" + hostname
metrics.send(metric4,
round(docker_mem_percent, 3),
host=k,
nodename=hostname)
print metric4, round(docker_mem_percent,
3), "host=" + k, "nodename=" + hostname
metrics.send(metric7, docker_io_read, host=k, nodename=hostname)
print metric7, docker_io_read, "host=" + k, "nodename=" + hostname
metrics.send(metric8, docker_io_write, host=k, nodename=hostname)
print metric8, docker_io_write, "host=" + k, "nodename=" + hostname
print "======================================================================================================="
except Exception, e:
print "Exception:", e, "At Line Number {}".format(
sys.exc_info()[-1].tb_lineno)
'net.in_packet.error': in_packet_error_eno2,
'net.out_packet.error': out_packet_error_eno2,
'net.qlen': metric_value_2.get('ifOutQLen_eno2'),
}
metric_server_t1 = {
'net.tcp.in_segment.error': in_tcp_seg_error,
'net.tcp.out_segment.reset': out_tcp_resets,
'net.retrans': percent_retrans,
'net.in_ip_fragment': in_frag_percent,
'net.in_out_fragment': out_frag_percent
}
metrics = potsdb.Client('175.126.104.46',
port=4343,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
#server related metrics push
def search(values, searchFor):
for k in values:
if searchFor in k:
#print k,values.get(k),"\n"
# return k,values.get(k)
list_1 = k.split("_")
#print "list is",list_1[0],list_1[1]
list_1[0] = "net." + list_1[0]
print "%s %s %s" % (list_1[0], values.get(k), hostname)
#metrics.send(list_1[0],values.get(k),server_ip=list_1[1])
# - logging configuration
logging.basicConfig()
logger = logging.getLogger('netclient')
logger.setLevel(logging.DEBUG)
host_ip = ""
host_port = 9033 #random
mode = "stream"
nums = 0
db_host = 'localhost'
db_port = 4242
metrics = potsdb.Client(db_host, port=db_port, qsize=25, mps=20)
badsite_keywords = {
"facebook", "twitter", "reddit", "netflix", "fb", "messenger"
}
path = ""
def encodePath(path):
pathStrs = ""
for pe in path:
pstr = pe.name
if pe.key:
for k, v in pe.key.iteritems():
pstr += "[" + str(k) + "=" + str(v) + "]"
pathStrs = pathStrs + "." + pstr
for element in countdict:
countdict[element] = len(countdict[element])
topk_keys = sorted(countdict, key = countdict.get, reverse = True)[:kDomainsToStore]
for key in topk_keys:
for value in set(dict[key]):
#(Metric name, value, tags ...)
metrics.send('Domains', dict[key].count(value), timestamp= item["timeStamp"], domain = key, serverId = item["serverId"])
"""
Main
"""
if __name__ == "__main__":
metrics = potsdb.Client(config["openTSDB"]["address"], config["openTSDB"]["port"])
answersPerSecond = AnswerPerSecondListener(r, ['AnswersPerSecond'])
answersPerSecond.start()
queriesPerSecond = QueriesPerSecondListener(r, ['QueriesPerSecond'])
queriesPerSecond.start()
queriesSummary = QueriesSummaryListener(r, ['QueriesSummary'])
queriesSummary.start()
topK = TopKListener(r, ['TopK'])
##topK.start() #Not used channel
topKWithIP = TopKWithIPListener(r, ['TopKWithIP'])
topKWithIP.start()
def execute(configIn=None, loggerIn=None):
""" Execute main script for DTN-RM Agent output preparation """
config, logger = getDefaultConfigAgent(COMPONENT, configIn, loggerIn)
if not config.getboolean(COMPONENT, 'enabled'):
logger.info('Component is not enabled in configuration.')
return
success, outConfig = getNetdataConfig()
if not success:
logger.warning('Failure receiving netdata configuration file. Exiting')
exit(1)
workDir = config.get('agent', 'PRIVATE_DIR') + "/DTNRM/"
agentDB = contentDB(logger=logger, config=config)
# TODO Implement enabled option and debug per each component.
# if not config.getboolean(COMPONENT, 'enabled'):
# return # http://vocms025.cern.ch:19999/api/v1/alarms?active&_=1508422746160
agents = makeGETRequest("http://localhost:19999/api/v1/alarms?active")
out = {
'netdata.warnings': 0,
'netdata.critical': 0,
'sense.nodes': 1,
'sense.vlansInUse': 0,
'sense.vlansFree': 0,
'sense.vlansTotal': 0,
'sense.agentError': 0,
'sense.agentWarning': 0,
'sense.agent.recurringactions.status': 0,
'sense.agent.recurringactions.runtime': 0,
'sense.agent.ruler.status': 0,
'sense.agent.ruler.runtime': 0
}
for _dKey, dVals in agents[1]['alarms'].items():
if dVals['status'] == 'WARNING':
out['netdata.warnings'] += 1
elif dVals['status'] == 'CRITICAL':
out['netdata.critical'] += 1
agentOut = agentDB.getFileContentAsJson(workDir + "/latest-out.json")
for _interf, interfDict in agentOut['NetInfo'].items():
if 'vlan_range' in interfDict:
lowR, highR = interfDict['vlan_range'].split(',')
total = int(highR) - int(lowR)
if total < 0:
# TODO print error
out['sense.agentError'] += 1
else:
out['sense.vlansTotal'] += total
out['sense.vlansInUse'] = len(interfDict['vlans'])
free = out['sense.vlansTotal'] - out['sense.vlansInUse']
if free < 0:
# TODO: print error
out['sense.agentError'] += 1
else:
out['sense.vlansFree'] = free
if 'destination' in outConfig:
splVal = outConfig['destination'].split(':')
if len(splVal) == 3:
destHostname = splVal[1]
destPort = splVal[2]
elif len(splVal) == 2:
destHostname = splVal[0]
destPort = splVal[1]
else:
print 'FAILURE. Was expecting protocol:ip:port or ip:port... Got Value: %s' % outConfig[
'destination']
return
else:
print 'FAILURE. Backend is not configured'
return
metrics = potsdb.Client(destHostname,
port=destPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
for key, value in out.items():
metrics.send("monit.%s" % key, value, **outConfig['tags'])
# waits for all outstanding metrics to be sent and background thread closes
metrics.wait()
#!/usr/bin/env python
'''
Author: Veerndra.K
Description: A simple example snippet, sends metrics to OpenTSDB
Requried Modules: potsdb (pip install potsdb)
'''
import potsdb
import time
tsdbIp = "127.0.0.1"
tsdbPort = 4343
interval = 5
try:
metrics = potsdb.Client(tsdbIp,
tsdbPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
ts = int(time.time())
metrics.send("sample.test", 1, host="my-host")
print "sample.test 1 host=my-host"
metrics.wait()
time.sleep(1)
except:
pass
time.sleep(5)
def metrics_send(n, tn, v):
name = "%s.%s.%s.%s" % (NAME_PREFIX, SPATH, tn, n)
print "sending metric %s = %d " % (name, v)
metrics = potsdb.Client(METRICS_HOST)
metrics.send(name, v)
metrics.wait()
#!/usr/bin/python
import potsdb
import sys
# set this to the hostname of the host running opentsdb
METRICS_HOST = 'node71'
if (len(sys.argv) != 3):
print "usage: %s %s %s" (sys.argv[0], "<metric_name>", "<metric_value>")
sys.exit(-1)
MNAME = sys.argv[1]
MVALUE = int(sys.argv[2])
print "sending metric %s = %d " % (MNAME, MVALUE)
metrics = potsdb.Client(METRICS_HOST)
metrics.send(MNAME, MVALUE)
metrics.wait()
print "done"
sys.exit(0)
# current datetime
current_datetime = datetime.datetime.utcnow()
for key in api_keys:
try:
# get server list
https_connection = http.client.HTTPSConnection("api.leaseweb.com")
https_headers = { 'x-lsw-auth': api_keys[key]}
https_connection.request("GET", "/bareMetals/v2/servers", headers=https_headers)
https_response = https_connection.getresponse()
https_data = https_response.read()
json_servers = json.loads(https_data.decode("utf-8"))
# connect to opentsdb
metrics = potsdb.Client(opentsdb_server, port=opentsdb_port,check_host=False)
except:
continue
if not 'servers' in json_servers:
print('LeaseWeb', key, 'returned no data')
continue
# for each server
for bareMetal in json_servers['servers']:
try:
# get id and name
bareMetalId = bareMetal['id']
serverName = bareMetal['contract']['internalReference']
# get datatraffic
def main():
host_mem_size = round(
(float(read_file(file_locations['memory'], 1, 1))) / (2**10), 3)
try:
sleeptime = 3
a = get_all_values()
time.sleep(sleeptime)
b = get_all_values()
metrics = potsdb.Client(tsdbIp,
port=tsdbPort,
qsize=1000,
host_tag=True,
mps=100,
check_host=True)
for k, i in a.iteritems():
#memory
max_mem = b[k]['max_memory'] / pow(2, 20)
if max_mem > 22368547:
max_mem = host_mem_size
mem_usage = b[k]['usage_memory'] / pow(2, 20)
docker_mem_percent = (mem_usage * 100) / max_mem
mem_free = max_mem - mem_usage
#cpu
docker_cpu = ((b[k]['docker_cpu'] - a[k]['docker_cpu']) /
sleeptime) * (pow(10, -9))
sys_cpu = ((b[k]['sys_cpu'] - a[k]['sys_cpu']) / sleeptime) * (pow(
10, -2))
docker_cpu_percent = (docker_cpu / sys_cpu) * 100
#disk
docker_io_read = round(
((b[k]['docker_io_read'] - a[k]['docker_io_read']) / sleeptime)
/ pow(2, 20), 3)
docker_io_write = round(
((b[k]['docker_io_write'] - a[k]['docker_io_write']) /
sleeptime) / pow(2, 20), 3)
#interface
intface_stat1 = a[k]['intface_stat']
intface_stat2 = b[k]['intface_stat']
rx = round(((float(intface_stat2[1]) - float(intface_stat1[1])) /
sleeptime) / pow(2, 10),
4) # - float(intface_stat1[1])
tx = round(((float(intface_stat2[9]) - float(intface_stat1[9])) /
sleeptime) / pow(2, 10),
4) # - float(intface_stat1[9])
rx_drop = round(
((float(intface_stat2[4]) - float(intface_stat1[4])) /
sleeptime)) # - float(intface_stat1[4])
tx_drop = round(
((float(intface_stat2[12]) - float(intface_stat1[12])) /
sleeptime))
interface = intface_stat2[0].split(":")[0]
metrics.send(metric6,
rx,
interface=interface,
host=k,
nodename=hostname)
metrics.send(metric5,
tx,
interface=interface,
host=k,
nodename=hostname)
metrics.send(metric12,
rx_drop,
interface=interface,
host=k,
nodename=hostname)
metrics.send(metric13,
tx_drop,
interface=interface,
host=k,
nodename=hostname)
print metric6, rx, "interface=" + interface, "host=" + k, "nodename=", hostname
print metric5, tx, "interface=" + interface + "host=" + k, "nodename=" + hostname
print metric12, rx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname
print metric13, tx_drop, "interface=" + interface, "host=" + k, "nodename=" + hostname
metrics.send(metric1,
round(docker_cpu_percent, 3),
host=k,
nodename=hostname)
print metric1, round(docker_cpu_percent,
3), "host=" + k, "nodename=" + hostname
metrics.send(metric2, mem_usage, host=k, nodename=hostname)
print metric2, mem_usage, "host=" + k, "nodename=" + hostname
metrics.send(metric9, mem_free, host=k, nodename=hostname)
print metric9, mem_free, "host=" + k, "nodename=" + hostname
metrics.send(metric3, max_mem, host=k, nodename=hostname)
print metric3, max_mem, "host=" + k, "nodename=" + hostname
metrics.send(metric4,
round(docker_mem_percent, 3),
host=k,
nodename=hostname)
print metric4, round(docker_mem_percent,
3), "host=" + k, "nodename=" + hostname
metrics.send(metric7, docker_io_read, host=k, nodename=hostname)
print metric7, docker_io_read, "host=" + k, "nodename=" + hostname
metrics.send(metric8, docker_io_write, host=k, nodename=hostname)
print metric8, docker_io_write, "host=" + k, "nodename=" + hostname
print "======================================================================================================="
except:
print "=============TRACKBACK============="
print traceback.format_exc()
def __init__(self):
self.client = potsdb.Client('localhost')
# - logging configuration
logging.basicConfig()
logger = logging.getLogger('test-client')
logger.setLevel(logging.DEBUG)
host_ip = "localhost"
host_port = 80050
mode = "stream"
db_host = '127.0.0.1'
#db_port = 8086
db_port = 4242
metrics = potsdb.Client(db_host, port=db_port)
#client = InfluxDBClient('localhost', 8086, 'root', 'root', 'example')
#client.create_database('example')
def encodePath(path):
pathStrs = ""
for pe in path:
pstr = pe.name
if pe.key:
for k, v in pe.key.iteritems():
pstr += "[" + str(k) + "=" + str(v) + "]"
pathStrs = pathStrs + "." + pstr
return pathStrs[1:]
def mapToJson(k,v,data):
config.openTSDB['host'] = config.openTSDB['host'] if os.getenv(
'OPENTSDB_HOST', 'None') == 'None' else os.getenv('OPENTSDB_HOST',
'127.0.0.1')
config.openTSDB['port'] = config.openTSDB['port'] if os.getenv(
'OPENTSDB_PORT', 'None') == 'None' else int(
os.getenv('OPENTSDB_PORT', '4242'))
config.GlobalAuthData[
'username'] = config.GlobalAuthData['username'] if os.getenv(
'AUTH_USERNAME', 'None') == 'None' else os.getenv('AUTH_USERNAME', '')
config.GlobalAuthData[
'password'] = config.GlobalAuthData['password'] if os.getenv(
'AUTH_PASSWORD', 'None') == 'None' else os.getenv('AUTH_PASSWORD', '')
metrics = potsdb.Client(config.openTSDB['host'],
port=config.openTSDB['port'],
check_host=True)
now_last = datetime.now().date()
utc_date = None
if '-' in num_env:
utc_date = datetime(
*(time.strptime(num_env, '%Y-%m-%d')[0:6])) - timedelta(days=1)
else:
utc_date = datetime.now() - timedelta(days=int(num_env) + 1)
jqls = [{
"new": config.Jira['jql_new']
}, {
"closed": config.Jira['jql_closed']
def __init__(self, *args, **kwargs):
super(RyuToOpentsdb, self).__init__(*args, **kwargs)
self.datapaths = {}
self.collector_thread = hub.spawn(self.run_stats_collector)
self.metrics = potsdb.Client(os.getenv('OTSDB_HOST', 'opentsdb'))
self.metric_prefix = os.getenv('OTSDB_METRIC_PREFIX', 'oftester')
def _get_client(**kwargs):
my_kwargs = {"port": PORT, "check_host": False, "test_mode": True}
my_kwargs.update(kwargs)
return potsdb.Client(HOST, **my_kwargs)
