def __init__(self):
from kafka import SimpleProducer, KafkaClient
from kafka.common import LeaderNotAvailableError
self.kafka_client = KafkaClient(config.KAFKA_SERVER)
self.kafka = SimpleProducer(self.kafka_client)
self.head = HeadBuilder("db", "type", "tom", config.DB_NAME)
try:
self.kafka.send_messages(config.KAFKA_TOPIC, b"creating topic")
except LeaderNotAvailableError:
time.sleep(1)
def __init__(self):
oids = config.SNMP_OIDS
utils.check_dir()
self.head = HeadBuilder("_index", "_type", "_id", config.ES_INDEX)
self.event_id = 1
self.event_log = open(utils.event_log + ".es", "w", encoding="utf-8")
for oid in oids:
file_name = utils.log_dir + "/" + oid._name() + "_log.es"
log = open(file_name, "w", encoding="utf-8")
self.files[oid.oid_id] = log
self.indices[oid._name()] = 0
def __init__(self):
utils.check_dir()
self.head = HeadBuilder("_index", "_type", "_id", config.DB_NAME)
self.session_log = open(utils.session_log+".es", "w", encoding="utf-8")
self.session_info()
self.cpu_sys_id = 1
self.cpu_system_log = open(utils.cpu_sys_log+".es", "w", encoding="utf-8")
self.cpu_proc_id = 1
self.cpu_proc_log = open(utils.cpu_proc_log+".es", "w", encoding="utf-8")
self.mem_sys_id = 1
self.mem_system_log = open(utils.mem_sys_log+".es", "w", encoding="utf-8")
self.mem_proc_id = 1
self.mem_proc_log = open(utils.mem_proc_log+".es", "w", encoding="utf-8")
self.io_sys_id = 1
self.io_system_log = open(utils.io_sys_log+".es", "w", encoding="utf-8")
self.proc_info_id = 1
self.proc_info_log = open(utils.proc_info_log+".es", "w", encoding="utf-8")
self.event_id = 1
self.event_log = open(utils.event_log+".es", "w", encoding="utf-8")
def __init__(self):
utils.check_dir()
self.head = HeadBuilder("_index", "_type", "_id", config.DB_NAME)
self.session_log = open(utils.session_log + ".es",
"w",
encoding="utf-8")
self.session_info()
self.cpu_sys_id = 1
self.cpu_system_log = open(utils.cpu_sys_log + ".es",
"w",
encoding="utf-8")
self.cpu_proc_id = 1
self.cpu_proc_log = open(utils.cpu_proc_log + ".es",
"w",
encoding="utf-8")
self.mem_sys_id = 1
self.mem_system_log = open(utils.mem_sys_log + ".es",
"w",
encoding="utf-8")
self.mem_proc_id = 1
self.mem_proc_log = open(utils.mem_proc_log + ".es",
"w",
encoding="utf-8")
self.io_sys_id = 1
self.io_system_log = open(utils.io_sys_log + ".es",
"w",
encoding="utf-8")
self.proc_info_id = 1
self.proc_info_log = open(utils.proc_info_log + ".es",
"w",
encoding="utf-8")
self.event_id = 1
self.event_log = open(utils.event_log + ".es", "w", encoding="utf-8")
def __init__(self):
from kafka import SimpleProducer, KafkaClient
from kafka.common import LeaderNotAvailableError
self.kafka_client = KafkaClient(config.KAFKA_SERVER)
self.kafka = SimpleProducer(self.kafka_client)
for oid in config.SNMP_OIDS:
self.indices[oid._name()] = 0
self.head = HeadBuilder("db", "type", "tom", config.DB_NAME)
try:
self.kafka.send_messages(config.KAFKA_TOPIC, b"creating topic")
except LeaderNotAvailableError:
time.sleep(1)
class ESFileLogger:
def __init__(self):
utils.check_dir()
self.head = HeadBuilder("_index", "_type", "_id", config.DB_NAME)
self.session_log = open(utils.session_log + ".es",
"w",
encoding="utf-8")
self.session_info()
self.cpu_sys_id = 1
self.cpu_system_log = open(utils.cpu_sys_log + ".es",
"w",
encoding="utf-8")
self.cpu_proc_id = 1
self.cpu_proc_log = open(utils.cpu_proc_log + ".es",
"w",
encoding="utf-8")
self.mem_sys_id = 1
self.mem_system_log = open(utils.mem_sys_log + ".es",
"w",
encoding="utf-8")
self.mem_proc_id = 1
self.mem_proc_log = open(utils.mem_proc_log + ".es",
"w",
encoding="utf-8")
self.io_sys_id = 1
self.io_system_log = open(utils.io_sys_log + ".es",
"w",
encoding="utf-8")
self.proc_info_id = 1
self.proc_info_log = open(utils.proc_info_log + ".es",
"w",
encoding="utf-8")
self.event_id = 1
self.event_log = open(utils.event_log + ".es", "w", encoding="utf-8")
def close(self):
self.cpu_system_log.close()
self.cpu_proc_log.close()
self.mem_system_log.close()
self.mem_proc_log.close()
self.io_system_log.close()
self.proc_info_log.close()
self.event_log.close()
def start(self):
pass
def commit(self):
pass
def session_info(self):
now = int(time.time() * 1000)
line = '{"index" : ' + self.head.create(
'session_info', 'session-' + str(now), now) + '}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "session_info", "_id" : "session-'+str(now)+'"}}\n'
line += bb.session_info()
self.session_log.write(line + "\n")
self.session_log.flush()
def cpu_sys(self, epoch, user_count, system_count, idle_count, percent):
"Logs CPU metrics at system level"
line = '{"index" : ' + self.head.create(
'system_cpu', 'cpu_sys_' + str(self.cpu_sys_id), epoch) + '}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "system_cpu", "_id" : "cpu_sys_'+str(self.cpu_sys_id)+'"}}\n'
line += bb.cpu_sys(epoch, user_count, system_count, idle_count,
percent)
self.cpu_sys_id += 1
self.cpu_system_log.write(line + "\n")
self.cpu_system_log.flush()
if config.PRINT_CONSOLE: print(line)
def cpu_proc(self, epoch, pid, priority, ctx_count, n_threads, cpu_user,
cpu_system, percent, pname):
"Logs CPU metrics at process level"
line = '{"index" : ' + self.head.create(
'process_cpu', 'cpu_proc_' + str(self.cpu_proc_id), epoch) + '}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "process_cpu", "_id" : "cpu_proc_'+str(self.cpu_proc_id)+'"}}\n'
line += bb.cpu_proc(epoch, pid, priority, ctx_count, n_threads,
cpu_user, cpu_system, percent, pname)
self.cpu_proc_id += 1
self.cpu_proc_log.write(line + "\n")
self.cpu_proc_log.flush()
if config.PRINT_CONSOLE: print(line)
def mem_sys(self, epoch, available, percent, used, free, swap_total,
swap_used, swap_free, swap_in, swap_out, swap_percent):
"Logs memory metrics at system level"
line = '{"index" : ' + self.head.create(
'system_memory', 'mem_sys_' + str(self.mem_sys_id), epoch) + '}\n'
# line = '{"index" : { "_index": "'+config.ES_INDEX+'", "_type" : "system_memory", "_id" : "mem_sys_'+str(self.mem_sys_id)+'"}}\n'
line += bb.mem_sys(epoch, available, percent, used, free, swap_total,
swap_used, swap_free, swap_in, swap_out,
swap_percent)
self.mem_sys_id += 1
self.mem_system_log.write(line + "\n")
self.mem_system_log.flush()
if config.PRINT_CONSOLE: print(line)
def mem_proc(self, epoch, pid, rss, vms, percent, pname):
"Logs memory metrics at process level"
line = '{"index" : ' + self.head.create(
'process_memory', 'mem_proc_' + str(self.mem_proc_id),
epoch) + '}\n'
# line = '{"index" : { "_index": "'+config.ES_INDEX+'", "_type": "process_memory", "_id": "mem_proc_'+str(self.mem_proc_id)+'"}}\n'
line += bb.mem_proc(epoch, pid, rss, vms, percent, pname)
self.mem_proc_id += 1
self.mem_proc_log.write(line + "\n")
self.mem_proc_log.flush()
if config.PRINT_CONSOLE: print(line)
def io_sys(self, epoch, bytes_sent, bytes_recv, packets_sent, packets_recv,
errin, errout, dropin, dropout):
"Print a line to console and to a file"
line = '{"index" : ' + self.head.create(
'system_io', 'io_sys_' + str(self.io_sys_id), epoch) + '}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "system_io", "_id" : "io_sys_'+str(self.io_sys_id)+'"}}\n'
line += bb.io_sys(epoch, bytes_sent, bytes_recv, packets_sent,
packets_recv, errin, errout, dropin, dropout)
self.io_sys_id += 1
self.io_system_log.write(line + "\n")
self.io_system_log.flush()
if config.PRINT_CONSOLE: print(line)
def proc_error(self, epoch, pid, name):
"Print a line to console and to a file"
line = '{"index" : ' + self.head.create(
'event', 'proc_error_' + str(self.event_id), epoch) + '}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "event", "_id" : "proc_error_'+str(self.event_id)+'"}}\n'
line += bb.proc_error(epoch, pid, name)
self.event_id += 1
self.event_log.write(line + "\n")
self.event_log.flush()
if config.PRINT_CONSOLE: print(line)
def proc_info(self, epoch, pid, name):
"Print a line to console and to a file"
line = '{"index" : ' + self.head.create(
'process_info', 'proc_info_' + str(self.proc_info_id),
epoch) + '}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "process_info", "_id" : "proc_info_'+str(self.proc_info_id)+'"}}\n'
line += bb.proc_info(epoch, pid, name)
self.proc_info_id += 1
self.proc_info_log.write(line + "\n")
self.proc_info_log.flush()
if config.PRINT_CONSOLE: print(line)
class ESFileLogger:
files = {}
indices = {}
def __init__(self):
oids = config.SNMP_OIDS
utils.check_dir()
self.head = HeadBuilder("_index", "_type", "_id", config.ES_INDEX)
self.event_id = 1
self.event_log = open(utils.event_log + ".es", "w", encoding="utf-8")
for oid in oids:
file_name = utils.log_dir + "/" + oid._name() + "_log.es"
log = open(file_name, "w", encoding="utf-8")
self.files[oid.oid_id] = log
self.indices[oid._name()] = 0
def close(self):
for file in self.files:
file.close()
self.event_log.close()
def start(self, epoch):
epoch *= 1000 #this converts it into milliseconds
head = '{"index" : ' + self.head.create('event', 'event_' + str(self.event_id), epoch) + '}\n'
body = '{"time" : ' + str(epoch) + ', "session_info" : "start"}'
line = head + body
self.event_log.write(line + "\n")
self.event_log.flush()
self.event_id += 1
if config.PRINT_CONSOLE: print(line)
def stop(self, epoch):
epoch *= 1000
head = '{"index" : ' + self.head.create('event', 'event_' + str(self.event_id), epoch) + '}\n'
body = '{"time" : ' + str(epoch) + ', "session_info" : "stop"}'
line = head + body
self.event_log.write(line + "\n")
self.event_log.flush()
self.event_id += 1
if config.PRINT_CONSOLE: print(line)
def value(self, epoch, oid, name, value):
epoch *= 1000
# name = oid._name()
name = name.replace(' ', '_')
index = self.indices[name]
index += 1
self.indices[name] = index
str_value = str(value)
if not oid.numeric or not utils.is_number(str_value):
str_value = '"'+str_value+'"'
# else:
# if not utils.is_number(str_value):
# self.error(epoch, "Numeric OID "+oid.oid+" produced non-numeric value:"+str_value)
# return
head = '{"index" : ' + self.head.create(name, name + '_' + str(index), epoch) + '}\n'
body = '{"time" : ' + str(epoch) + ', "target" : "' + str(oid.target()) + '", ' + \
'"target_name" : "' + str(oid.target_name) + '", "oid" : "' + str(oid.oid_id) + '", ' + \
'"oid_name" : "' + str(name) + '", "value" : ' + str_value + '}'
line = head + body
log = self.files[oid.oid_id]
log.write(line + "\n")
log.flush()
if config.PRINT_CONSOLE: print(line)
def error(self, epoch, description):
epoch *= 1000
head = '{"index" : ' + self.head.create('event', 'event_' + str(self.event_id), epoch) + '}\n'
body = '{"time" : ' + str(epoch) + ', "error" : "' + description + '"}'
line = head + body
self.event_log.write(line + "\n")
self.event_log.flush()
self.event_id += 1
if config.PRINT_CONSOLE: print(line)
class ESFileLogger:
def __init__(self):
utils.check_dir()
self.head = HeadBuilder("_index", "_type", "_id", config.DB_NAME)
self.session_log = open(utils.session_log+".es", "w", encoding="utf-8")
self.session_info()
self.cpu_sys_id = 1
self.cpu_system_log = open(utils.cpu_sys_log+".es", "w", encoding="utf-8")
self.cpu_proc_id = 1
self.cpu_proc_log = open(utils.cpu_proc_log+".es", "w", encoding="utf-8")
self.mem_sys_id = 1
self.mem_system_log = open(utils.mem_sys_log+".es", "w", encoding="utf-8")
self.mem_proc_id = 1
self.mem_proc_log = open(utils.mem_proc_log+".es", "w", encoding="utf-8")
self.io_sys_id = 1
self.io_system_log = open(utils.io_sys_log+".es", "w", encoding="utf-8")
self.proc_info_id = 1
self.proc_info_log = open(utils.proc_info_log+".es", "w", encoding="utf-8")
self.event_id = 1
self.event_log = open(utils.event_log+".es", "w", encoding="utf-8")
def close(self):
self.cpu_system_log.close()
self.cpu_proc_log.close()
self.mem_system_log.close()
self.mem_proc_log.close()
self.io_system_log.close()
self.proc_info_log.close()
self.event_log.close()
def start(self): pass
def commit(self): pass
def session_info(self):
now = int(time.time()*1000)
line = '{"index" : '+self.head.create('session_info', 'session-'+str(now), now)+'}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "session_info", "_id" : "session-'+str(now)+'"}}\n'
line += bb.session_info()
self.session_log.write(line + "\n")
self.session_log.flush()
def cpu_sys(self, epoch, user_count, system_count, idle_count, percent):
"Logs CPU metrics at system level"
line = '{"index" : '+self.head.create('system_cpu', 'cpu_sys_'+str(self.cpu_sys_id), epoch)+'}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "system_cpu", "_id" : "cpu_sys_'+str(self.cpu_sys_id)+'"}}\n'
line += bb.cpu_sys(epoch, user_count, system_count, idle_count, percent)
self.cpu_sys_id += 1
self.cpu_system_log.write(line + "\n")
self.cpu_system_log.flush()
if config.PRINT_CONSOLE: print(line)
def cpu_proc(self, epoch, pid, priority, ctx_count, n_threads, cpu_user, cpu_system, percent, pname):
"Logs CPU metrics at process level"
line = '{"index" : '+self.head.create('process_cpu', 'cpu_proc_'+str(self.cpu_proc_id), epoch)+'}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "process_cpu", "_id" : "cpu_proc_'+str(self.cpu_proc_id)+'"}}\n'
line += bb.cpu_proc(epoch, pid, priority, ctx_count, n_threads, cpu_user, cpu_system, percent, pname)
self.cpu_proc_id += 1
self.cpu_proc_log.write(line + "\n")
self.cpu_proc_log.flush()
if config.PRINT_CONSOLE: print(line)
def mem_sys(self, epoch, available, percent, used, free,
swap_total, swap_used, swap_free, swap_in, swap_out, swap_percent):
"Logs memory metrics at system level"
line = '{"index" : '+self.head.create('system_memory', 'mem_sys_'+str(self.mem_sys_id), epoch)+'}\n'
# line = '{"index" : { "_index": "'+config.ES_INDEX+'", "_type" : "system_memory", "_id" : "mem_sys_'+str(self.mem_sys_id)+'"}}\n'
line += bb.mem_sys(epoch, available, percent, used, free, swap_total, swap_used, swap_free, swap_in, swap_out, swap_percent)
self.mem_sys_id += 1
self.mem_system_log.write(line + "\n")
self.mem_system_log.flush()
if config.PRINT_CONSOLE: print(line)
def mem_proc(self, epoch, pid, rss, vms, percent, pname):
"Logs memory metrics at process level"
line = '{"index" : '+self.head.create('process_memory', 'mem_proc_'+str(self.mem_proc_id), epoch)+'}\n'
# line = '{"index" : { "_index": "'+config.ES_INDEX+'", "_type": "process_memory", "_id": "mem_proc_'+str(self.mem_proc_id)+'"}}\n'
line += bb.mem_proc(epoch, pid, rss, vms, percent, pname)
self.mem_proc_id += 1
self.mem_proc_log.write(line + "\n")
self.mem_proc_log.flush()
if config.PRINT_CONSOLE: print(line)
def io_sys(self, epoch, bytes_sent, bytes_recv, packets_sent, packets_recv, errin, errout, dropin, dropout):
"Print a line to console and to a file"
line = '{"index" : '+self.head.create('system_io', 'io_sys_'+str(self.io_sys_id), epoch)+'}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "system_io", "_id" : "io_sys_'+str(self.io_sys_id)+'"}}\n'
line += bb.io_sys(epoch, bytes_sent, bytes_recv, packets_sent, packets_recv, errin, errout, dropin, dropout)
self.io_sys_id += 1
self.io_system_log.write(line + "\n")
self.io_system_log.flush()
if config.PRINT_CONSOLE: print(line)
def proc_error(self, epoch, pid, name):
"Print a line to console and to a file"
line = '{"index" : '+self.head.create('event', 'proc_error_'+str(self.event_id), epoch)+'}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "event", "_id" : "proc_error_'+str(self.event_id)+'"}}\n'
line += bb.proc_error(epoch, pid, name)
self.event_id += 1
self.event_log.write(line + "\n")
self.event_log.flush()
if config.PRINT_CONSOLE: print(line)
def proc_info(self, epoch, pid, name):
"Print a line to console and to a file"
line = '{"index" : '+self.head.create('process_info', 'proc_info_'+str(self.proc_info_id), epoch)+'}\n'
# line = '{"index" : { "_index" : "'+config.ES_INDEX+'", "_type" : "process_info", "_id" : "proc_info_'+str(self.proc_info_id)+'"}}\n'
line += bb.proc_info(epoch, pid, name)
self.proc_info_id += 1
self.proc_info_log.write(line + "\n")
self.proc_info_log.flush()
if config.PRINT_CONSOLE: print(line)
class KafkaLogger:
def __init__(self):
from kafka import SimpleProducer, KafkaClient
from kafka.common import LeaderNotAvailableError
self.kafka_client = KafkaClient(config.KAFKA_SERVER)
self.kafka = SimpleProducer(self.kafka_client)
self.head = HeadBuilder("db", "type", "tom", config.DB_NAME)
try:
self.kafka.send_messages(config.KAFKA_TOPIC, b"creating topic")
except LeaderNotAvailableError:
time.sleep(1)
def close(self):
self.kafka.stop(0)
self.kafka_client.close()
def start(self):
pass
def commit(self):
pass
def session_info(self):
body = bb.session_info()
now = int(time.time() * 1000)
header = self.head.create('info', config.TOM, now)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
def cpu_sys(self, epoch, user_count, system_count, idle_count, percent):
"Logs CPU metrics at system level"
body = bb.cpu_sys(epoch, user_count, system_count, idle_count, percent)
header = self.head.create('system_cpu', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
def cpu_proc(self, epoch, pid, priority, ctx_count, n_threads, cpu_user,
cpu_system, percent, pname):
"Logs CPU metrics at process level"
epoch *= 1000 #this converts it into milliseconds
body = bb.cpu_proc(epoch, pid, priority, ctx_count, n_threads,
cpu_user, cpu_system, percent, pname)
header = self.head.create('process_cpu', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
def mem_sys(self, epoch, available, percent, used, free, swap_total,
swap_used, swap_free, swap_in, swap_out, swap_percent):
"Logs memory metrics at system level"
epoch *= 1000 #this converts it into milliseconds
body = bb.mem_sys(epoch, available, percent, used, free, swap_total,
swap_used, swap_free, swap_in, swap_out,
swap_percent)
header = self.head.create('system_memory', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
def mem_proc(self, epoch, pid, rss, vms, percent, pname):
"Logs memory metrics at process level"
epoch *= 1000 #this converts it into milliseconds
body = bb.mem_proc(epoch, pid, rss, vms, percent, pname)
header = self.head.create('process_memory', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
def io_sys(self, epoch, bytes_sent, bytes_recv, packets_sent, packets_recv,
errin, errout, dropin, dropout):
"Print a line to console and to a file"
epoch *= 1000 #this converts it into milliseconds
body = bb.io_sys(epoch, bytes_sent, bytes_recv, packets_sent,
packets_recv, errin, errout, dropin, dropout)
header = self.head.create('system_io', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
def proc_error(self, epoch, pid, name):
"Print a line to console and to a file"
epoch *= 1000 #this converts it into milliseconds
body = bb.proc_error(epoch, pid, name)
header = self.head.create('event', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
def proc_info(self, epoch, pid, name):
"Print a line to console and to a file"
epoch *= 1000 #this converts it into milliseconds
body = bb.proc_info(epoch, pid, name)
header = self.head.create('process_info', config.TOM, epoch)
msg = '{"header": ' + header + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
# if config.PRINT_CONSOLE: print(reply)
# from kafka import SimpleProducer, KafkaClient
#
# # To send messages synchronously
# kafka = KafkaClient("192.168.2.79:9092")
# producer = SimpleProducer(kafka)
#
# # Note that the application is responsible for encoding messages to type str
# producer.send_messages("my-topic", b"some message")
# producer.send_messages("my-topic", b"this method", b"is variadic")
#
# # Send unicode message
# producer.send_messages("my-topic", u'你怎么样?'.encode('utf-8'))
#
# # To send messages asynchronously
# # WARNING: current implementation does not guarantee message delivery on failure!
# # messages can get dropped! Use at your own risk! Or help us improve with a PR!
# producer = SimpleProducer(kafka, async=True)
# producer.send_messages("my-topic", b"async message")
#
# # To wait for acknowledgements
# # ACK_AFTER_LOCAL_WRITE : server will wait till the data is written to
# # a local log before sending response
# # ACK_AFTER_CLUSTER_COMMIT : server will block until the message is committed
# # by all in sync replicas before sending a response
# producer = SimpleProducer(kafka, async=False,
# req_acks=SimpleProducer.ACK_AFTER_LOCAL_WRITE,
# ack_timeout=2000)
#
# response = producer.send_messages("my-topic", b"another message")
#
# if response:
# print(response[0].error)
# print(response[0].offset)
#
# # To send messages in batch. You can use any of the available
# # producers for doing this. The following producer will collect
# # messages in batch and send them to Kafka after 20 messages are
# # collected or every 60 seconds
# # Notes:
# # * If the producer dies before the messages are sent, there will be losses
# # * Call producer.stop() to send the messages and cleanup
# producer = SimpleProducer(kafka, batch_send=True,
# batch_send_every_n=20,
# batch_send_every_t=60)
class KafkaJsonLogger:
indices = {}
def __init__(self):
from kafka import SimpleProducer, KafkaClient
from kafka.common import LeaderNotAvailableError
self.kafka_client = KafkaClient(config.KAFKA_SERVER)
self.kafka = SimpleProducer(self.kafka_client)
for oid in config.SNMP_OIDS:
self.indices[oid._name()] = 0
self.head = HeadBuilder("db", "type", "tom", config.DB_NAME)
try:
self.kafka.send_messages(config.KAFKA_TOPIC, b"creating topic")
except LeaderNotAvailableError:
time.sleep(1)
def close(self):
self.kafka.stop(0)
self.kafka_client.close()
def start(self, epoch):
head = self.head.create('info', config.TOM, epoch)
body = '{"description" : "started ('+ config.SESSION_NAME + ')"'+', "value" : '+str(config.SESSION_ID)+'}'
msg = '{"header": '+ head + ', "body":'+ body+'}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
def stop(self, epoch):
head = self.head.create('info', config.TOM, epoch)
body = '{"description" : "stopped ('+ config.SESSION_NAME + ')"'+', "value" : '+str(config.SESSION_ID)+'}'
msg = '{"header": '+ head + ', "body":'+ body+'}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
def value(self, epoch, oid, name, value):
# name = oid._name()
name = name.replace(' ', '_')
index = self.indices[name]
index += 1
self.indices[name] = index
is_error = False
str_value = str(value)
if oid.numeric and not utils.is_number(str_value):
is_error = True
head = self.head.create(name, oid.target_name, epoch)
body = '{"target" : "' + str(oid.target()) + '", ' + \
'"oid" : "' + str(oid.oid_id)
if is_error:
body += '", "error" : "' + str_value + '"}'
else:
body += '", "value" : ' + str_value + '}'
msg = '{"header": '+ head + ', "body":'+ body+'}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
def error(self, epoch, description):
head = self.head.create('info', config.TOM, epoch)
body = '{"error" : "'+ description + '"}'
msg = '{"header": '+ head + ', "body":'+ body+'}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
class KafkaJsonLogger:
indices = {}
def __init__(self):
from kafka import SimpleProducer, KafkaClient
from kafka.common import LeaderNotAvailableError
self.kafka_client = KafkaClient(config.KAFKA_SERVER)
self.kafka = SimpleProducer(self.kafka_client)
for oid in config.SNMP_OIDS:
self.indices[oid._name()] = 0
self.head = HeadBuilder("db", "type", "tom", config.DB_NAME)
try:
self.kafka.send_messages(config.KAFKA_TOPIC, b"creating topic")
except LeaderNotAvailableError:
time.sleep(1)
def close(self):
self.kafka.stop(0)
self.kafka_client.close()
def start(self, epoch):
head = self.head.create('info', config.TOM, epoch)
body = '{"description" : "started (' + config.SESSION_NAME + ')"' + ', "value" : ' + str(
config.SESSION_ID) + '}'
msg = '{"header": ' + head + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
def stop(self, epoch):
head = self.head.create('info', config.TOM, epoch)
body = '{"description" : "stopped (' + config.SESSION_NAME + ')"' + ', "value" : ' + str(
config.SESSION_ID) + '}'
msg = '{"header": ' + head + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
def value(self, epoch, oid, name, value):
# name = oid._name()
name = name.replace(' ', '_')
index = self.indices[name]
index += 1
self.indices[name] = index
is_error = False
str_value = str(value)
if oid.numeric and not utils.is_number(str_value):
is_error = True
head = self.head.create(name, oid.target_name, epoch)
body = '{"target" : "' + str(oid.target()) + '", ' + \
'"oid" : "' + str(oid.oid_id)
if is_error:
body += '", "error" : "' + str_value + '"}'
else:
body += '", "value" : ' + str_value + '}'
msg = '{"header": ' + head + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)
def error(self, epoch, description):
head = self.head.create('info', config.TOM, epoch)
body = '{"error" : "' + description + '"}'
msg = '{"header": ' + head + ', "body":' + body + '}'
self.kafka.send_messages(config.KAFKA_TOPIC, msg.encode("utf8"))
if config.PRINT_CONSOLE: print(msg)