def crash_test(string):
print color.INFO("Opening persistent TCP connection for diagnostics")
sock_mem.connect((HOST, PORT_MEM))
get_mem_start()
print color.HEADER("Initial crash test")
burst_size = BURST_SIZE * 10
ARP_burst(burst_size, 0)
UDP_burst(burst_size, 0)
ICMP_flood(burst_size, 0)
httperf(burst_size, 0)
time.sleep(BURST_INTERVAL)
return get_mem()
def fire_bursts(func, sub_test_name, lead_out=3):
name_tag = "<" + sub_test_name + ">"
print color.HEADER(test_name + " initiating " + sub_test_name)
membase_start = func()
mem_base = membase_start
# Track heap behavior
increases = 0
decreases = 0
constant = 0
for i in range(0, BURST_COUNT):
print color.INFO(name_tag), " Run ", i + 1
memi = func()
if memi > mem_base:
memincrease = memi - mem_base
increases += 1
elif memi == mem_base:
memincrease = 0
constant += 1
else:
memincrease = 0
decreases += 1
# We want to know how much each burst increases memory relative to the last burst
mem_base = memi
if memincrease > acceptable_increase:
print color.WARNING(
name_tag), "Memory increased by ", memincrease, "b, ", float(
memincrease) / BURST_SIZE, "pr. packet \n"
else:
print color.OK(name_tag), "Memory increase ", memincrease, "b \n"
# Memory can decrease, we don't care about that
# if memincrease > 0:
# mem_base += memincrease
print color.INFO(
name_tag
), "Heap behavior: ", "+", increases, ", -", decreases, ", ==", constant
print color.INFO(name_tag), "Done. Checking for liveliness"
if memory_increase(lead_out, membase_start) > acceptable_increase:
print color.FAIL(sub_test_name + " failed ")
return False
print color.PASS(sub_test_name + " succeeded ")
return True
def misc_working(misc_tests):
global test_count
test_count += len(misc_tests)
if len(misc_tests) == 0:
return 0
if ("misc" in args.skip):
print pretty.WARNING("Misc test skipped")
return 0
print pretty.HEADER("Building " + str(len(misc_tests)) + " misc")
fail_count = 0
for test in misc_tests:
build = test.start().wait_status()
fail_count += 1 if build else 0
return fail_count
def filter_tests(all_tests, arguments):
""" Will figure out which tests are to be run
Arguments:
all_tests (list of Test obj): all processed test objects
arguments (argument object): Contains arguments from argparse
returns:
list: All Test objects that are to be run
"""
print pretty.HEADER("Filtering tests")
# Strip trailing slashes from paths
add_args = [ x.rstrip("/") for x in arguments.tests ]
skip_args = [ x.rstrip("/") for x in arguments.skip ]
print pretty.INFO("Tests to run"), ", ".join(add_args)
# 1) Add tests to be run
# If no tests specified all are run
if not add_args:
tests_added = [ x for x in all_tests if x.type_ in test_types ]
else:
tests_added = [ x for x in all_tests
if x.type_ in add_args
or x.category_ in add_args
or x.name_ in add_args ]
# 2) Remove tests defined by the skip argument
print pretty.INFO("Tests marked skip on command line"), ", ".join(skip_args)
skipped_tests = [ x for x in tests_added
if x.type_ in skip_args
or x.category_ in skip_args
or x.name_ in skip_args
or x.skip_]
# Print all the skipped tests
print_skipped(skipped_tests)
fin_tests = [ x for x in tests_added if x not in skipped_tests ]
print pretty.INFO("Accepted tests"), ", ".join([x.name_ for x in fin_tests])
return fin_tests
def misc_working():
global test_count
if ("misc" in args.skip):
print pretty.WARNING("Misc test skipped")
return 0
misc_dir = 'misc'
dirs = os.walk(misc_dir).next()[1]
dirs.sort()
print pretty.HEADER("Building " + str(len(dirs)) + " misc")
test_count += len(dirs)
fail_count = 0
for directory in dirs:
misc = misc_dir + "/" + directory
print "Building misc ", misc
build = Test(misc, command = ['./test.sh'], name = directory).start().wait_status()
run = 0 #TODO: Make a 'test' folder for each miscellanous test, containing test.py, vm.json etc.
fail_count += 1 if build or run else 0
return fail_count
def stress_test():
"""Perform stresstest"""
global test_count
test_count += 1
if ("stress" in args.skip):
print pretty.WARNING("Stress test skipped")
return 0
if (not validate_tests.validate_test("stress")):
raise Exception("Stress test failed validation")
print pretty.HEADER("Starting stress test")
stress = Test("stress", clean = args.clean).start()
if (stress and args.fail):
print pretty.FAIL("Stress test failed")
sys.exit(stress)
return 1 if stress.wait_status() else 0
def stress_test(stress_tests):
"""Perform stresstest"""
global test_count
test_count += len(stress_tests)
if len(stress_tests) == 0:
return 0
if ("stress" in args.skip):
print pretty.WARNING("Stress test skipped")
return 0
if (not validate_tests.validate_test("stress")):
raise Exception("Stress test failed validation")
print pretty.HEADER("Starting stress test")
for test in stress_tests:
test.start()
for test in stress_tests:
return 1 if test.wait_status() else 0
def crash_test(string):
print(color.INFO("Opening persistent TCP connection for diagnostics"))
sock_mem.connect((HOST, PORT_MEM))
mem_before = get_mem_start()
if mem_before <= 0:
print(color.FAIL("Initial memory reported as " + str(mem_before)))
return False
if not heap_verified:
print(color.FAIL("Heap behavior was not verified as expected. "))
return False
print(color.HEADER("Initial crash test"))
burst_size = BURST_SIZE * 10
ARP_burst(burst_size, 0)
UDP_burst(burst_size, 0)
ICMP_flood(burst_size, 0)
httperf(burst_size, 0)
time.sleep(BURST_INTERVAL)
mem_after = get_mem()
print(color.INFO("Crash test complete. Memory in use: "), mem_after)
return mem_after >= memuse_at_start
color.INFO(
"There are {0} sockets in use, waiting for value to drop below {1}"
.format(time_wait_proc, socket_limit)))
time.sleep(7)
# Add custom event-handlers
vm.on_output("Heap functioning as expected", heap_ok)
vm.on_output("Ready to start", crash_test)
vm.on_output("Ready for ARP", ARP)
vm.on_output("Ready for UDP", UDP)
vm.on_output("Ready for ICMP", ICMP)
vm.on_output("Ready for TCP", TCP)
vm.on_output("Ready to end", check_vitals)
if len(sys.argv) > 1:
thread_timeout = int(sys.argv[1])
if len(sys.argv) > 3:
BURST_COUNT = int(sys.argv[2])
BURST_SIZE = int(sys.argv[3])
print(color.HEADER(test_name + " initializing"))
print(color.INFO(name_tag), "configured for ", BURST_COUNT, "bursts of",
BURST_SIZE, "packets each")
if len(sys.argv) > 4:
vm.boot(timeout=thread_timeout, image_name=str(sys.argv[4]))
else:
vm.cmake().boot(thread_timeout).clean()
def filter_tests(all_tests, arguments):
""" Will figure out which tests are to be run
Arguments:
all_tests (list of Test obj): all processed test objects
arguments (argument object): Contains arguments from argparse
returns:
tuple: (All Test objects that are to be run, skipped_tests)
"""
print pretty.HEADER("Filtering tests")
# Strip trailing slashes from paths
add_args = [x.rstrip("/") for x in arguments.tests]
skip_args = [x.rstrip("/") for x in arguments.skip]
print pretty.INFO("Tests to run"), ", ".join(add_args)
# 1) Add tests to be run
# If no tests specified all are run
if not add_args:
tests_added = [x for x in all_tests if x.type_ in test_types]
else:
tests_added = [
x for x in all_tests if x.type_ in add_args
or x.category_ in add_args or x.name_ in add_args
]
# Deal with specific properties
add_properties = list(
set(add_args).intersection(all_tests[0].properties_.keys()))
for test in all_tests:
for argument in add_properties:
if test.properties_[argument] and test not in tests_added:
tests_added.append(test)
# 2) Remove tests defined by the skip argument
print pretty.INFO("Tests marked skip on command line"), ", ".join(
skip_args)
skipped_tests = [
x for x in tests_added if x.type_ in skip_args
or x.category_ in skip_args or x.name_ in skip_args or x.skip_
]
# Deal with specific properties
skip_properties = list(
set(skip_args).intersection(all_tests[0].properties_.keys()))
for test in tests_added:
for argument in skip_properties:
if test.properties_[argument] and test not in skipped_tests:
test.skip_ = True
test.skip_reason_ = "Test marked skip on command line"
skipped_tests.append(test)
# Print all the skipped tests
print_skipped(skipped_tests)
fin_tests = [x for x in tests_added if x not in skipped_tests]
print pretty.INFO("Accepted tests"), ", ".join(
[x.name_ for x in fin_tests])
return (fin_tests, skipped_tests)
def integration_tests(tests):
""" Function that runs the tests that are passed to it.
Filters out any invalid tests before running
Arguments:
tests: List containing test objects to be run
Returns:
integer: Number of tests that failed
"""
if len(tests) == 0:
return 0
time_sensitive_tests = [
x for x in tests if x.properties_["time_sensitive"]
]
tests = [x for x in tests if x not in time_sensitive_tests]
# Print info before starting to run
print pretty.HEADER("Starting " + str(len(tests)) + " integration test(s)")
for test in tests:
print pretty.INFO("Test"), "starting", test.name_
if time_sensitive_tests:
print pretty.HEADER("Then starting " + str(len(time_sensitive_tests)) +
" time sensitive integration test(s)")
for test in time_sensitive_tests:
print pretty.INFO("Test"), "starting", test.name_
processes = []
fail_count = 0
global test_count
test_count += len(tests) + len(time_sensitive_tests)
# Find number of cpu cores
if args.parallel:
num_cpus = args.parallel
else:
num_cpus = multiprocessing.cpu_count()
# Collect test results
print pretty.HEADER(
"Collecting integration test results, on {0} cpu(s)".format(num_cpus))
# Run a maximum number of parallell tests equal to cpus available
while tests or processes: # While there are tests or processes left
try:
processes.append(
tests.pop(0).start()) # Remove test from list after start
except IndexError:
pass # All tests done
while (len(processes) == num_cpus) or not tests:
# While there are a maximum of num_cpus to process
# Or there are no more tests left to start we wait for them
for p in list(processes): # Iterate over copy of list
if p.proc_.poll() is not None:
fail_count += 1 if p.wait_status() else 0
processes.remove(p)
time.sleep(1)
if not processes and not tests:
break
# Exit early if any tests failed
if fail_count and args.fail:
print pretty.FAIL(str(fail_count) + "integration tests failed")
sys.exit(fail_count)
# Start running the time sensitive tests
for test in time_sensitive_tests:
process = test.start()
fail_count += 1 if process.wait_status() else 0
if fail_count and args.fail:
print pretty.FAIL(str(fail_count) + "integration tests failed")
sys.exit(fail_count)
return fail_count
