def wait_for_tw(): print color.INFO("Waiting for sockets to clear TIME_WAIT stage") socket_limit = 11500 time_wait_proc = 30000 while time_wait_proc > socket_limit: output = subprocess.check_output(('netstat', '-anlt')) output = output.split('\n') time_wait_proc = 0 for line in output: if "TIME_WAIT" in line: time_wait_proc += 1 print color.INFO( "There are {0} sockets in use, waiting for value to drop below {1}" .format(time_wait_proc, socket_limit)) time.sleep(7)
def check_vitals(): print color.INFO("Checking vital signs") mem = get_mem() diff = mem - memuse_at_start pages = diff / PAGE_SIZE if diff % PAGE_SIZE != 0: print color.WARNING("Memory increase was not a multple of page size.") wait_for_tw() return False print color.INFO("Memory use at test end:"), mem, "bytes" print color.INFO( "Memory difference from test start:" ), memuse_at_start, "bytes (Diff:", diff, "b == ", pages, "pages)" sock_mem.close() vm.stop() wait_for_tw() return True
def fire_bursts(func, sub_test_name, lead_out=3): name_tag = "<" + sub_test_name + ">" print color.HEADER(test_name + " initiating " + sub_test_name) membase_start = func() mem_base = membase_start # Track heap behavior increases = 0 decreases = 0 constant = 0 for i in range(0, BURST_COUNT): print color.INFO(name_tag), " Run ", i + 1 memi = func() if memi > mem_base: memincrease = memi - mem_base increases += 1 elif memi == mem_base: memincrease = 0 constant += 1 else: memincrease = 0 decreases += 1 # We want to know how much each burst increases memory relative to the last burst mem_base = memi if memincrease > acceptable_increase: print color.WARNING( name_tag), "Memory increased by ", memincrease, "b, ", float( memincrease) / BURST_SIZE, "pr. packet \n" else: print color.OK(name_tag), "Memory increase ", memincrease, "b \n" # Memory can decrease, we don't care about that # if memincrease > 0: # mem_base += memincrease print color.INFO( name_tag ), "Heap behavior: ", "+", increases, ", -", decreases, ", ==", constant print color.INFO(name_tag), "Done. Checking for liveliness" if memory_increase(lead_out, membase_start) > acceptable_increase: print color.FAIL(sub_test_name + " failed ") return False print color.PASS(sub_test_name + " succeeded ") return True
def crash_test(): print color.INFO("Opening persistent TCP connection for diagnostics") sock_mem.connect((HOST, PORT_MEM)) get_mem_start() print color.HEADER("Initial crash test") burst_size = BURST_SIZE * 10 ARP_burst(burst_size, 0) UDP_burst(burst_size, 0) ICMP_flood(burst_size, 0) httperf(burst_size, 0) time.sleep(BURST_INTERVAL) return get_mem()
def get_mem(): name_tag = "<" + test_name + "::get_mem>" try: # We expect this socket to allready be opened time.sleep(1) sock_mem.send("memsize\n") received = sock_mem.recv(1000).rstrip() except Exception as e: print color.FAIL( name_tag), "Python socket failed while getting memsize: ", e return False print color.INFO( name_tag), "Current VM memory usage reported as ", received return int(received)
def memory_increase(lead_time, expected_memuse=memuse_at_start): name_tag = "<" + test_name + "::memory_increase>" if lead_time: print color.INFO( name_tag ), "Checking for memory increase after a lead time of ", lead_time, "s." # Give the VM a chance to free up resources before asking time.sleep(lead_time) use = get_mem() increase = use - expected_memuse percent = 0.0 if (increase): percent = float(increase) / expected_memuse if increase > acceptable_increase: print color.WARNING(name_tag), "Memory increased by ", percent, "%." print "(", expected_memuse, "->", use, ",", increase, "b increase, but no increase expected.)" else: print color.OK(name_tag + "Memory constant, no leak detected") return increase
time_wait_proc += 1 print color.INFO( "There are {0} sockets in use, waiting for value to drop below {1}" .format(time_wait_proc, socket_limit)) time.sleep(7) # Add custom event-handlers vm.on_output("Ready to start", crash_test) vm.on_output("Ready for ARP", ARP) vm.on_output("Ready for UDP", UDP) vm.on_output("Ready for ICMP", ICMP) vm.on_output("Ready for TCP", TCP) vm.on_output("Ready to end", check_vitals) # Boot the VM, taking a timeout as parameter timeout = BURST_COUNT * 20 if len(sys.argv) > 1: timeout = int(sys.argv[1]) if len(sys.argv) > 3: BURST_COUNT = int(sys.argv[2]) BURST_SIZE = int(sys.argv[3]) print color.HEADER(test_name + " initializing") print color.INFO( name_tag), "Doing", BURST_COUNT, "bursts of", BURST_SIZE, "packets each" vm.make().boot(timeout)