def test_proc_error_csv(self):
csv = CSVFileLogger()
csv.proc_error(11111, 22, "epic fail")
csv.proc_error(11111, 9758, "fail")
csv.proc_error(11112, 7364, "little fail")
csv.close()
actual = open(utils.event_log + ".csv").read()
expected = pkg_resources.resource_string(
self.__module__, 'expected_events.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_proc_info_csv(self):
csv = CSVFileLogger()
csv.proc_info(11111, 22, "proc1")
csv.proc_info(11111, 9758, "proc2")
csv.proc_info(11111, 7364, "proc4")
csv.proc_info(11111, 3332, "proc3")
csv.close()
actual = open(utils.proc_info_log + ".csv").read()
expected = pkg_resources.resource_string(
self.__module__, 'expected_proc_info.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_cpu_sys_csv(self):
# self.assertEqual("a", "b")
csv = CSVFileLogger()
csv.cpu_sys(0, 1, 1, 1, 1)
csv.cpu_sys(1, 3, 2, 5, 6)
csv.cpu_sys(3, 22, 99, 11, 4)
csv.cpu_sys(5, 155, 122, 12, 22)
csv.close()
actual = open(utils.cpu_sys_log + ".csv").read()
expected = pkg_resources.resource_string(
self.__module__, 'expected_cpu_sys.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_proc_error_csv(self):
csv = CSVFileLogger()
csv.proc_error(11111, 22, "epic fail")
csv.proc_error(11111, 9758, "fail")
csv.proc_error(11112, 7364, "little fail")
csv.close()
actual = open(utils.event_log+".csv").read()
expected = pkg_resources.resource_string(self.__module__, 'expected_events.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_proc_info_csv(self):
csv = CSVFileLogger()
csv.proc_info(11111, 22, "proc1")
csv.proc_info(11111, 9758, "proc2")
csv.proc_info(11111, 7364, "proc4")
csv.proc_info(11111, 3332, "proc3")
csv.close()
actual = open(utils.proc_info_log+".csv").read()
expected = pkg_resources.resource_string(self.__module__, 'expected_proc_info.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_cpu_sys_csv(self):
# self.assertEqual("a", "b")
csv = CSVFileLogger()
csv.cpu_sys(0, 1, 1, 1, 1)
csv.cpu_sys(1, 3, 2, 5, 6)
csv.cpu_sys(3, 22, 99, 11, 4)
csv.cpu_sys(5, 155, 122, 12, 22)
csv.close()
actual = open(utils.cpu_sys_log+".csv").read()
expected = pkg_resources.resource_string(self.__module__, 'expected_cpu_sys.csv').decode('utf8')
t_assert.equal(actual, expected)
def init():
global loggers
global proc_poller
global cpu_poller
global mem_poller
global io_poller
loggers = []
if (config.ES_FILE_ENABLED): loggers.append(ESFileLogger())
if (config.ES_NW_ENABLED): loggers.append(ESNetLogger())
if (config.MYSQL_ENABLED): loggers.append(MySqlLogger())
if (config.CSV_ENABLED): loggers.append(CSVFileLogger())
if (config.KAFKA_ENABLED): loggers.append(KafkaLogger())
proc_poller = ProcPoller(loggers)
cpu_poller = CPUPoller(proc_poller, loggers)
mem_poller = MemPoller(proc_poller, loggers)
io_poller = IOPoller(loggers)
def test_mem_proc_csv(self):
csv = CSVFileLogger()
csv.mem_proc(0, 1, 11, 15, 5, "p1")
csv.mem_proc(0, 2, 1, 3, 2, "p2")
csv.mem_proc(0, 5432, 21, 33, 9, "p3")
csv.mem_proc(5, 1, 22, 11, 3, "p1")
csv.mem_proc(5, 5432, 7, 55, 7, "p3")
csv.mem_proc(66, 1, 11, 15, 5, "p1")
csv.mem_proc(66, 2, 11, 0, 22, "p2")
csv.mem_proc(66, 5432, 212, 334, 44, "p3")
csv.close()
actual = open(utils.mem_proc_log + ".csv").read()
#self.__module__ gives current package name
expected = pkg_resources.resource_string(
self.__module__, 'expected_mem_proc.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_io_sys_csv(self):
csv = CSVFileLogger()
csv.io_sys(11111, 22, 22, 34, 43, 11, 11, 5, 3)
csv.io_sys(22222, 55, 23, 44, 34, 23, 17, 15, 4)
csv.io_sys(22233, 65, 23, 777, 44, 28, 18, 35, 5)
csv.io_sys(25555, 78, 44, 1911, 53, 99434, 43, 43, 21)
csv.close()
actual = open(utils.io_sys_log + ".csv").read()
expected = pkg_resources.resource_string(
self.__module__, 'expected_io_sys.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_cpu_proc_csv(self):
csv = CSVFileLogger()
csv.cpu_proc(0, 1, 1, 1, 1, 1, 1, 1, "p1")
csv.cpu_proc(0, 2, 1, 3, 4, 2, 3, 1, "p2")
csv.cpu_proc(0, 3, 2, 122, 7, 5, 8, 11, "p3")
csv.cpu_proc(10, 1, 1, 1, 1, 1, 1, 1, "p1")
csv.cpu_proc(10, 2, 1, 3, 4, 2, 3, 1, "p2")
csv.cpu_proc(10, 3, 2, 122, 7, 5, 8, 11, "p3")
csv.cpu_proc(20, 1, 1, 5, 1, 4, 3, 2, "p1")
csv.cpu_proc(20, 3, 2, 555, 7, 11, 55, 32, "p3")
csv.close()
actual = open(utils.cpu_proc_log + ".csv").read()
expected = pkg_resources.resource_string(
self.__module__, 'expected_cpu_proc.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_mem_sys_csv(self):
csv = CSVFileLogger()
csv.mem_sys(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)
csv.mem_sys(10, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2)
csv.mem_sys(12, 34, 654, 24, 33, 23, 442, 1, 13, 21, 44)
csv.mem_sys(15, 3445, 345, 345, 44, 745, 367, 32, 1111, 33, 55)
csv.mem_sys(33, 33, 453, 998, 347, 976, 8544, 45, 5555, 66, 33)
csv.close()
actual = open(utils.mem_sys_log + ".csv").read()
expected = pkg_resources.resource_string(
self.__module__, 'expected_mem_sys.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_cpu_proc_csv(self):
csv = CSVFileLogger()
csv.cpu_proc(0, 1, 1, 1, 1, 1, 1, 1, "p1")
csv.cpu_proc(0, 2, 1, 3, 4, 2, 3, 1, "p2")
csv.cpu_proc(0, 3, 2, 122, 7, 5, 8, 11, "p3")
csv.cpu_proc(10, 1, 1, 1, 1, 1, 1, 1, "p1")
csv.cpu_proc(10, 2, 1, 3, 4, 2, 3, 1, "p2")
csv.cpu_proc(10, 3, 2, 122, 7, 5, 8, 11, "p3")
csv.cpu_proc(20, 1, 1, 5, 1, 4, 3, 2, "p1")
csv.cpu_proc(20, 3, 2, 555, 7, 11, 55, 32, "p3")
csv.close()
actual = open(utils.cpu_proc_log+".csv").read()
expected = pkg_resources.resource_string(self.__module__, 'expected_cpu_proc.csv').decode('utf8')
t_assert.equal(actual, expected)
# pass <- pass is NOP in Python
self.proc_poller.handle_process_poll_error(epoch, proc)
def poll(self):
# int() converts argument to integer (string or float), in this case the float time
epoch = int(time.time() * 1000)
self.poll_system(epoch)
self.proc_poller.check_processes(epoch)
for pid in config.PROCESS_LIST:
if pid == "-": return
if pid == "*":
for proc in psutil.process_iter():
self.proc_poller.check_info(epoch, proc)
self.poll_process(epoch, proc)
return
processes = self.proc_poller.get_processes(pid)
for proc in processes:
self.poll_process(epoch, proc)
after = int(time.time() * 1000)
if __name__ == "__main__":
file = CSVFileLogger(True)
mem_poller = MemPoller(ProcPoller(), file)
while (True):
mem_poller.poll()
time.sleep(1)
file.close()
def poll_process(self, epoch, proc):
try:
pid = proc.pid
pname = self.proc_poller.get_name(proc)
priority = proc.nice()
#status is a text indicator such as "running". ignoring for now.
# status = proc.status()
ctx_switches = proc.num_ctx_switches()
ctx_count = ctx_switches.voluntary + ctx_switches.involuntary
n_threads = proc.num_threads()
cpu_times = proc.cpu_times()
cpu_user = cpu_times.user
cpu_system = cpu_times.system
cpu_percent = proc.cpu_percent()
for logger in self.loggers:
logger.cpu_proc(epoch, pid, priority, ctx_count, n_threads, cpu_user, cpu_system, cpu_percent, pname)
except (psutil.NoSuchProcess, psutil.AccessDenied):
#if the process has disappeared, we get an exception and ignore it
#pass <- pass is NOP in Python
self.proc_poller.handle_process_poll_error(epoch, proc)
if __name__ == "__main__":
csv = CSVFileLogger()
es = ESFileLogger()
proc = ProcPoller(csv, es)
cpu_poller = CPUPoller(proc, csv, es)
while (True):
cpu_poller.poll()
time.sleep(config.INTERVAL)
csv.close()
self.loggers = loggers
def poll_system(self, epoch):
#TODO: per NIC data
#TODO: disk data
net_counters = psutil.net_io_counters()
bytes_sent = net_counters.bytes_sent
bytes_recv = net_counters.bytes_recv
packets_sent = net_counters.packets_sent
packets_recv = net_counters.packets_recv
errin = net_counters.errin
errout = net_counters.errout
dropin = net_counters.dropin
dropout = net_counters.dropout
for logger in self.loggers:
logger.io_sys(epoch, bytes_sent, bytes_recv, packets_sent,
packets_recv, errin, errout, dropin, dropout)
def poll(self):
# int() converts argument to integer (string or float), in this case the float time
epoch = int(time.time() * 1000)
self.poll_system(epoch)
if __name__ == "__main__":
file = CSVFileLogger()
io_poller = IOPoller(file)
while (True):
io_poller.poll()
time.sleep(1)
file.close()
def test_mem_sys_csv(self):
csv = CSVFileLogger()
csv.mem_sys(0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)
csv.mem_sys(10, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2)
csv.mem_sys(12, 34, 654, 24, 33, 23, 442, 1, 13, 21, 44)
csv.mem_sys(15, 3445, 345, 345, 44, 745, 367, 32, 1111, 33, 55)
csv.mem_sys(33, 33, 453, 998, 347, 976, 8544, 45, 5555, 66, 33)
csv.close()
actual = open(utils.mem_sys_log+".csv").read()
expected = pkg_resources.resource_string(self.__module__, 'expected_mem_sys.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_io_sys_csv(self):
csv = CSVFileLogger()
csv.io_sys(11111, 22, 22, 34, 43, 11, 11, 5, 3)
csv.io_sys(22222, 55, 23, 44, 34, 23, 17, 15, 4)
csv.io_sys(22233, 65, 23, 777, 44, 28, 18, 35, 5)
csv.io_sys(25555, 78, 44, 1911, 53, 99434, 43, 43, 21)
csv.close()
actual = open(utils.io_sys_log+".csv").read()
expected = pkg_resources.resource_string(self.__module__, 'expected_io_sys.csv').decode('utf8')
t_assert.equal(actual, expected)
def test_mem_proc_csv(self):
csv = CSVFileLogger()
csv.mem_proc(0, 1, 11, 15, 5, "p1")
csv.mem_proc(0, 2, 1, 3, 2, "p2")
csv.mem_proc(0, 5432, 21, 33, 9, "p3")
csv.mem_proc(5, 1, 22, 11, 3, "p1")
csv.mem_proc(5, 5432, 7, 55, 7, "p3")
csv.mem_proc(66, 1, 11, 15, 5, "p1")
csv.mem_proc(66, 2, 11, 0, 22, "p2")
csv.mem_proc(66, 5432, 212, 334, 44, "p3")
csv.close()
actual = open(utils.mem_proc_log+".csv").read()
#self.__module__ gives current package name
expected = pkg_resources.resource_string(self.__module__, 'expected_mem_proc.csv').decode('utf8')
t_assert.equal(actual, expected)
pid = proc.pid
pname = self.proc_poller.get_name(proc)
priority = proc.nice()
#status is a text indicator such as "running". ignoring for now.
# status = proc.status()
ctx_switches = proc.num_ctx_switches()
ctx_count = ctx_switches.voluntary + ctx_switches.involuntary
n_threads = proc.num_threads()
cpu_times = proc.cpu_times()
cpu_user = cpu_times.user
cpu_system = cpu_times.system
cpu_percent = proc.cpu_percent()
for logger in self.loggers:
logger.cpu_proc(epoch, pid, priority, ctx_count, n_threads,
cpu_user, cpu_system, cpu_percent, pname)
except (psutil.NoSuchProcess, psutil.AccessDenied):
#if the process has disappeared, we get an exception and ignore it
#pass <- pass is NOP in Python
self.proc_poller.handle_process_poll_error(epoch, proc)
if __name__ == "__main__":
csv = CSVFileLogger()
es = ESFileLogger()
proc = ProcPoller(csv, es)
cpu_poller = CPUPoller(proc, csv, es)
while (True):
cpu_poller.poll()
time.sleep(config.INTERVAL)
csv.close()
class IOPoller:
def __init__(self, loggers):
self.loggers = loggers
def poll_system(self, epoch):
#TODO: per NIC data
#TODO: disk data
net_counters = psutil.net_io_counters()
bytes_sent = net_counters.bytes_sent
bytes_recv = net_counters.bytes_recv
packets_sent = net_counters.packets_sent
packets_recv = net_counters.packets_recv
errin = net_counters.errin
errout = net_counters.errout
dropin = net_counters.dropin
dropout = net_counters.dropout
for logger in self.loggers:
logger.io_sys(epoch, bytes_sent, bytes_recv, packets_sent, packets_recv, errin, errout, dropin, dropout)
def poll(self):
# int() converts argument to integer (string or float), in this case the float time
epoch = int(time.time()*1000)
self.poll_system(epoch)
if __name__ == "__main__":
file = CSVFileLogger()
io_poller = IOPoller(file)
while (True):
io_poller.poll()
time.sleep(1)
file.close()
