def test_insert_cpu(self):
'''unit test for insert_network
Insert a few values and make sure they can be retrieved'''
hosts = ["client", "client", "server"]
identifiers = ["cpu1", "cpu2", "cpu1"]
times = [
time.time(),
time.time() + 15.347,
time.time() + 53.421
]
user_pcts = [23.7, 32.4, 77.8]
sys_pcts = [10.2, 23.2, 8.3]
wait_pcts = [1.1, 0.0, 2.1]
irq_pcts = [2.3, 4.3, 0.1]
idle_pcts = [0.0, 0.5, 0.0]
total_pcts = [33.9, 55.6, 86.1]
expected_rows = []
con = create_perf_db(":memory:")
for i in range(len(hosts)):
insert_cpu(con, hosts[i], identifiers[i], times[i],
user_pcts[i], sys_pcts[i], wait_pcts[i], irq_pcts[i],
idle_pcts[i], total_pcts[i], commit=False)
expected_rows.append((hosts[i], identifiers[i],
epoch_to_sql(times[i]), user_pcts[i], sys_pcts[i],
wait_pcts[i], irq_pcts[i], idle_pcts[i], total_pcts[i]))
con.commit()
cur = con.cursor()
query = "SELECT host, cpu_identifier, time, user_pct, sys_pct, wait_pct, irq_pct, idle_pct, total_pct from Cpu;"
cur.execute(query)
results = cur.fetchall()
self.assertEqual(len(expected_rows), len(results))
for row in expected_rows:
self.assertIn(row, results)
def test_insert_disk(self):
'''unit test for insert_disk
Insert a few vlues and make sure they can be retrieved'''
hosts = ["thirdparty", "thirdparty", "server"]
times = [
time.time(),
time.time() + 15.347,
time.time() + 53.421
]
disk_names = ["/dev/sda", "/dev/sdb", "/dev/hda"]
reads_per_secs = [5.3, 4.7, 6.8]
reads_kbps = [48.6, 983.2, 43.7]
writes_per_secs = [76.2, 53.4, 98.2]
writes_kbps = [123.4, 321.5, 1023.7]
expected_rows = []
con = create_perf_db(":memory:")
for i in range(len(hosts)):
insert_disk(con, hosts[i], times[i], disk_names[i],
reads_per_secs[i], reads_kbps[i], writes_per_secs[i],
writes_kbps[i], commit=False)
expected_rows.append((hosts[i], epoch_to_sql(times[i]),
disk_names[i], reads_per_secs[i], reads_kbps[i],
writes_per_secs[i], writes_kbps[i]))
con.commit()
cur = con.cursor()
query = "SELECT host, time, disk_name, reads_per_sec, reads_kbps, writes_per_sec, writes_kbps from Disk;"
cur.execute(query)
results = cur.fetchall()
self.assertEqual(len(expected_rows), len(results))
for row in expected_rows:
self.assertIn(row, results)
def test_insert_network(self):
'''unit test for insert_network
Insert a few values and make sure they can be retrieved'''
times = [
time.time(),
time.time() + 15.347,
time.time() + 53.421
]
directions = ["client_to_server",
"server_to_client", "server_to_client"]
payload_sizes = [500, 1024, 1460]
protocols = ["IP/TCP", "IP/TCP", "IP/TCP"]
con = create_perf_db(":memory:")
expected_rows = []
for i in range(len(times)):
insert_network(con, times[i], directions[i], payload_sizes[i],
protocols[i], commit=False)
expected_rows.append((epoch_to_sql(times[i]), directions[i],
payload_sizes[i], protocols[i]))
con.commit()
cur = con.cursor()
query = "SELECT time, direction, payloadlength, protocol FROM Network;"
cur.execute(query)
results = cur.fetchall()
self.assertEqual(len(expected_rows), len(results))
for row in expected_rows:
self.assertIn(row, results)
def insert_cpu(con,
host,
identifier,
time,
user_pct,
sys_pct,
wait_pct,
irq_pct,
idle_pct,
total_pct,
commit=True):
'''Inserts a row into the cpu table in the performance database.
Inputs:
con: a connection to the database: squlite3 connection
host: hostname ("server"): string
identifier: cpu identifier ("cpu 1"): string
time: time in the form of seconds since the epoch: float
user_pct: percentage of time spent on userland processes
sys_pct: percentage of time spent on system processes
wait_pct: percentage of time spent waiting on I/O
irq_pct: percentage of time spent processing interrupts
idle_pct: percentage of time spent idle
total_pct: percentage of time spent on (user+sys) processes
commit (optional): whether or not to commit the insertion to disk (default=True)
'''
con.execute(
"INSERT INTO " + CPU_TABLE + " Values(Null,?,?,?,?,?,?,?,?,?);",
(host, identifier, epoch_to_sql(time), user_pct, sys_pct, wait_pct,
irq_pct, idle_pct, total_pct))
if (commit):
con.commit()
return con
def test_insert_ram(self):
'''unit test for insert_ram
Insert a few values and make sure they can be retrieved'''
hosts = ["client", "thirdparty", "thirdparty"]
times = [
time.time(),
time.time() + 15.347,
time.time() + 53.421
]
used_kbs = [1024, 2047.0, 2046.3]
free_kbs = [0.0, 1.0, 0.7]
swap_totals = [4096.0, 8192.0, 8192.0]
swap_useds = [1234.5, 2102.7, 4212.0]
swap_frees = [2861.5, 6089.3, 3980.0]
expected_rows = []
con = create_perf_db(":memory:")
for i in range(len(hosts)):
insert_ram(con, hosts[i], times[i], used_kbs[i],
free_kbs[i], swap_totals[i], swap_useds[i],
swap_frees[i], commit=False)
expected_rows.append((hosts[i], epoch_to_sql(times[i]),
used_kbs[i], free_kbs[i], swap_totals[i], swap_useds[i],
swap_frees[i]))
con.commit()
cur = con.cursor()
query = "SELECT host, time, used_kb, free_kb, swap_total, swap_used, swap_free from Ram;"
cur.execute(query)
results = cur.fetchall()
self.assertEqual(len(expected_rows), len(results))
for row in expected_rows:
self.assertIn(row, results)
def insert_disk(con,
host,
time,
disk_name,
reads_per_sec,
reads_kbps,
writes_per_sec,
writes_kbps,
commit=True):
'''Inserts a row into the disk table in the performance database.
Inputs:
con: a connection to the database: squlite3 connection
host: hostname ("server"): string
time: time in the form of seconds since the epoch: float
disk_name: name of the disk ("/dev/sda"): string
reads_per_sec: number of reads per second: float
reads_kbps: amount of data read in kbps: float
writes_per_sec: number of writes per second: float
writes_kbps: amount of data written in kbps: float
commit (optional): whether or not to commit the insertion to disk (default=True)
'''
con.execute("INSERT INTO " + DISK_TABLE + " Values(Null, ?,?,?,?,?,?,?);",
(host, epoch_to_sql(time), disk_name, reads_per_sec,
reads_kbps, writes_per_sec, writes_kbps))
if (commit):
con.commit()
return con
def insert_ram(con,
host,
time,
used_kb,
free_kb,
swap_total,
swap_used,
swap_free,
commit=True):
'''Inserts a row into the ram table in the performance database.
Inputs:
con: a connection to the database: squlite3 connection
host: hostname ("server"): string
time: time in the form of seconds since the epoch: float
used_kb: amount of memory used: float
free_kb: amount of free memory: float
swap_total: amount of swap allocated by the system: float
swap_used: amount of swap in use: float
swap_free: amount of available swap: float
commit (optional): whether or not to commit the insertion to disk (default=True)
'''
con.execute("INSERT INTO " + RAM_TABLE + " Values(Null,?,?,?,?,?,?,?);",
(host, epoch_to_sql(time), used_kb, free_kb, swap_total,
swap_used, swap_free))
if (commit):
con.commit()
return con
def insert_network(con, time, direction, payload_size, protocol, commit=True):
'''Inserts a row into the network table in the performance database.
Inputs:
con: a connection to the database: squlite3 connection
time: time in the form of seconds since the epoch: float
direction: the direction of traffic ("client_to_server, etc): string
payload_size: number of bytes in the payload: integer
protocol: the network protocol ("IP/TCP", etc): string
commit (optional): whether or not to commit the insertion to disk (default=True)
'''
con.execute("INSERT INTO " + NETWORK_TABLE + " Values(Null,?,?,?,?);",
(epoch_to_sql(time), direction, payload_size, protocol))
if (commit):
con.commit()
return con
def test_write_to_database(self):
'''Unit test for write_to_database
Generate some records, write them to database
query database, ensure records in db'''
reader = nlp.NetworkLogReader(
StringIO.StringIO(TEST_NETWORK_LOG_FILE_1))
reader.add_handle(StringIO.StringIO(TEST_NETWORK_LOG_FILE_2))
reader.add_handle(StringIO.StringIO(TEST_NETWORK_LOG_FILE_3))
con = perfdb.create_perf_db(":memory:")
reader.write_to_database(con)
expected_entries = [{
"time": 1373902408.284399,
"direction": "client_to_thirdparty",
"payloadlength": 0,
"protocol": "IP/TCP"
}, {
"time": 1373902408.292883,
"direction": "thirdparty_to_client",
"payloadlength": 1460,
"protocol": "IP/TCP"
}, {
"time": 1373902411.754983,
"direction": "thirdparty_to_client",
"payloadlength": 0,
"protocol": "IP/TCP"
}]
expected_rows = []
for entry in expected_entries:
expected_rows.append(
(epoch_to_sql(entry["time"]), entry["direction"],
entry["payloadlength"], entry["direction"]))
query = "SELECT time, direction, payloadlength, direction FROM Network;"
cursor = con.cursor()
cursor.execute(query)
actual_rows = cursor.fetchall()
for entry in expected_rows:
self.assertIn(entry, actual_rows)
def test_write_to_database(self):
'''Unit test for write_to_database
Generate some records, write them to database
query database, ensure records in db'''
reader = clp.CpuLogReader(StringIO.StringIO(TEST_CPU_LOG_FILE_1), "client")
reader.add_handle(StringIO.StringIO(TEST_CPU_LOG_FILE_2), "server")
reader.add_handle(StringIO.StringIO(TEST_CPU_LOG_FILE_3), "thirdparty")
con = perfdb.create_perf_db(":memory:")
reader.write_to_database(con)
expected_entries = [
{
"host" : "client",
"time_epoch" : date_and_time_to_epoch("20130805", "11:38:56.377"),
"cpu_identifier" : "cpu_6",
"user_pct" : .72,
"sys_pct" : 0.0,
"wait_pct" : 0.0,
"irq_pct" : 0.0,
"idle_pct" : 99.28,
"total_pct" : .72
},
{
"host" : "server",
"time_epoch" : date_and_time_to_epoch("20130805", "11:39:00.505"),
"cpu_identifier" : "cpu_4",
"user_pct" : 2.94,
"sys_pct" : 0.74,
"wait_pct" : 0.0,
"irq_pct" : 0.0,
"idle_pct" : 96.32,
"total_pct" : 3.68
},
{
"host" : "thirdparty",
"time_epoch" : date_and_time_to_epoch("20130805", "11:39:06.009"),
"cpu_identifier" : "cpu_0",
"user_pct" : 0.0,
"sys_pct" : 0.0,
"wait_pct" : 2.17,
"irq_pct" : 0.0,
"idle_pct" : 97.83,
"total_pct" : 0.0
}
]
expected_rows = []
for entry in expected_entries:
expected_rows.append((entry["host"], epoch_to_sql(entry["time_epoch"]),
entry["cpu_identifier"],
entry["user_pct"], entry["sys_pct"],
entry["wait_pct"], entry["irq_pct"],
entry["idle_pct"], entry["total_pct"]))
query = "SELECT host, time, cpu_identifier, user_pct, sys_pct, wait_pct, irq_pct, idle_pct, total_pct from Cpu;"
cursor = con.cursor()
cursor.execute(query)
actual_rows = cursor.fetchall()
for entry in expected_rows:
self.assertIn(entry, actual_rows)
def test_write_to_database(self):
'''Unit test for write_to_database
Generate some records, write them to database
query database, ensure records in db'''
reader = rlp.RamLogReader(StringIO.StringIO(TEST_RAM_LOG_FILE_1),
"client")
reader.add_handle(StringIO.StringIO(TEST_RAM_LOG_FILE_2), "thirdparty")
reader.add_handle(StringIO.StringIO(TEST_RAM_LOG_FILE_3), "thirdparty")
con = perfdb.create_perf_db(":memory:")
reader.write_to_database(con)
expected_entries = [{
"host":
"client",
"time_epoch":
date_and_time_to_epoch("20130731", "11:33:33.003"),
"free_kb":
71368,
"used_kb":
940564,
"swap_total":
1046524,
"swap_used":
304800,
"swap_free":
741724
}, {
"host":
"thirdparty",
"time_epoch":
date_and_time_to_epoch("20130731", "11:33:42.003"),
"free_kb":
71244,
"used_kb":
940688,
"swap_total":
1046524,
"swap_used":
304800,
"swap_free":
741724
}, {
"host":
"thirdparty",
"time_epoch":
date_and_time_to_epoch("20130731", "11:33:57.003"),
"free_kb":
73352,
"used_kb":
938580,
"swap_total":
1046524,
"swap_used":
304800,
"swap_free":
741724
}]
expected_rows = []
for entry in expected_entries:
expected_rows.append(
(entry["host"], epoch_to_sql(entry["time_epoch"]),
entry["free_kb"], entry["used_kb"], entry["swap_total"],
entry["swap_used"], entry["swap_free"]))
query = "SELECT host, time, free_kb, used_kb, swap_total, swap_used, swap_free FROM Ram;"
cursor = con.cursor()
cursor.execute(query)
actual_rows = cursor.fetchall()
for entry in expected_rows:
self.assertIn(entry, actual_rows)
def test_write_to_database(self):
'''Unit test for write_to_database
Generate some records, write them to database
query database, ensure records in db'''
reader = dlp.DiskLogReader(StringIO.StringIO(TEST_DISK_LOG_FILE_1),
"client")
reader.add_handle(StringIO.StringIO(TEST_DISK_LOG_FILE_2), "server")
reader.add_handle(StringIO.StringIO(TEST_DISK_LOG_FILE_3),
"thirdparty")
con = perfdb.create_perf_db(":memory:")
reader.write_to_database(con)
expected_entries = [{
"host":
"client",
"time_epoch":
date_and_time_to_epoch("20130802", "16:07:52.568"),
"disk_name":
"sda",
"reads_per_sec":
240.84,
"reads_kbps":
3710.05,
"writes_per_sec":
4.68,
"writes_kbps":
59.24
}, {
"host":
"server",
"time_epoch":
date_and_time_to_epoch("20130802", "16:07:57.700"),
"disk_name":
"sda",
"reads_per_sec":
47.47,
"reads_kbps":
11297.47,
"writes_per_sec":
34.02,
"writes_kbps":
5901.90
}, {
"host":
"thirdparty",
"time_epoch":
date_and_time_to_epoch("20130802", "16:08:04.399"),
"disk_name":
"sdb",
"reads_per_sec":
301.87,
"reads_kbps":
2985.96,
"writes_per_sec":
1.56,
"writes_kbps":
31.20
}]
expected_rows = []
for entry in expected_entries:
expected_rows.append(
(entry["host"], epoch_to_sql(entry["time_epoch"]),
entry["disk_name"], entry["reads_per_sec"],
entry["reads_kbps"], entry["writes_per_sec"],
entry["writes_kbps"]))
query = "SELECT host, time, disk_name, reads_per_sec, reads_kbps, writes_per_sec, writes_kbps FROM Disk;"
cursor = con.cursor()
cursor.execute(query)
actual_rows = cursor.fetchall()
for entry in expected_rows:
self.assertIn(entry, actual_rows)