def boot(self, timeout = None, multiboot = True, kernel_args = "booted with vmrunner"):
# Check for sudo access, needed for tap network devices and the KVM module
if os.getuid() is not 0:
print color.FAIL("Call the script with sudo access")
sys.exit(1)
# Start the timeout thread
if (timeout):
self._timer = threading.Timer(timeout, self._on_timeout)
self._timer.start()
# Boot via hypervisor
try:
self._hyper.boot(multiboot, kernel_args + "/" + self._hyper.name())
except Exception as err:
print color.WARNING("Exception raised while booting ")
if (timeout): self._timer.cancel()
raise err
# Start analyzing output
while self._hyper.poll() == None and not self._exit_status:
line = self._hyper.readline()
print color.SUBPROC("<VM> "+line.rstrip())
# Look for event-triggers
for pattern, func in self._on_output.iteritems():
if re.search(pattern, line):
try:
res = func()
except Exception as err:
print color.WARNING("Exception raised in event callback: ")
print_exception()
res = False
self.stop().wait()
#NOTE: It can be 'None' without problem
if res == False:
self._exit_status = exit_codes["OUTSIDE_FAIL"]
self.exit(self._exit_status, color.FAIL(nametag + " VM-external test failed"))
# Now we either have an exit status from timer thread, or an exit status
# from the subprocess, or the VM was powered off by the external test.
# If the process didn't exit we need to stop it.
if (self.poll() == None):
self.stop()
self.wait()
if self._exit_status:
print color.WARNING(nametag + "Found non-zero exit status but process didn't end. ")
print color.FAIL(nametag + "Tests failed or program error")
print color.INFO(nametag),"Done running VM. Exit status: ", self._exit_status
sys.exit(self._exit_status)
else:
print color.SUCCESS(nametag + " VM exited with 0 exit status.")
print color.INFO(nametag), "Subprocess finished. Exiting with ", self._hyper.poll()
sys.exit(self._hyper.poll())
raise Exception("Unexpected termination")
def __init__(self, config, hyper = qemu):
self._exit_status = 0
self._config = config
self._on_success = lambda : self.exit(exit_codes["SUCCESS"], color.SUCCESS(nametag + " All tests passed"))
self._on_panic = lambda : self.exit(exit_codes["VM_FAIL"], color.FAIL(nametag + self._hyper.readline()))
self._on_timeout = lambda : self.exit(exit_codes["TIMEOUT"], color.FAIL(nametag + " Test timed out"))
self._on_output = {
"PANIC" : self._on_panic,
"SUCCESS" : self._on_success }
assert(issubclass(hyper, hypervisor))
self._hyper = hyper(config)
self._timer = None
self._on_exit_success = lambda : None
self._on_exit = lambda : None
def main():
global test_count
# Warned about skipped tests
# @note : doesn't warn if you use '-t a b c ...' to run specific tests
if args.skip:
for skip in args.skip:
print_skipped(skip, "marked skipped on command line")
test_categories = ["integration", "examples", "unit", "stress"]
if args.tests:
test_categories = [x for x in test_categories if x in args.tests or x == "integration"]
if args.skip:
test_categories = [x for x in test_categories if not x in args.skip]
integration = integration_tests() if "integration" in test_categories else 0
stress = stress_test() if "stress" in test_categories else 0
unit = unit_tests() if "unit" in test_categories else 0
examples = examples_working() if "examples" in test_categories else 0
status = max(integration, stress, unit, examples)
if (not test_count):
print "No tests selected"
exit(0)
if (status == 0):
print pretty.SUCCESS(str(test_count - status) + " / " + str(test_count)
+ " tests passed, exiting with code 0")
else:
print pretty.FAIL(str(status) + " / " + str(test_count) + " tests failed ")
sys.exit(status)
def integration_tests():
"""
Loops over all valid tests as defined by ./validate_all.py. Runs them one by one and gives an update of the statuses at the end.
"""
global test_count
valid = valid_tests()
if not valid:
print pretty.WARNING("Integration tests skipped")
return 0
test_count += len(valid)
print pretty.HEADER("Starting " + str(len(valid)) + " integration test(s)")
processes = []
fail_count = 0
for path in valid:
processes.append(Test(path, clean = args.clean).start())
# Collect test results
print pretty.HEADER("Collecting integration test results")
for p in processes:
fail_count += 1 if p.wait_status() else 0
# Exit early if any tests failed
if fail_count and args.fail:
print pretty.FAIL(str(fail_count) + "integration tests failed")
sys.exit(fail_count)
return fail_count
def main():
# Find leaf nodes
leaves = find_leaf_nodes()
# Populate test objects
all_tests = [Test(path) for path in leaves]
# Figure out which tests are to be run
test_categories_to_run = []
test_types_to_run = []
if args.tests:
for argument in args.tests:
if argument in test_categories and argument not in args.skip:
test_categories_to_run.append(argument)
elif argument in test_types and argument not in args.skip:
test_types_to_run.append(argument)
else:
print 'Test specified is not recognised, exiting'
sys.exit(1)
else:
test_types_to_run = test_types
if test_categories_to_run:
# This means that a specific category has been requested
specific_tests = [
test for test in all_tests
if test.category_ in test_categories_to_run
]
# Print which tests are skipped
print_skipped(specific_tests)
# Run the tests
integration = integration_tests(specific_tests)
else:
# Print which tests are skipped
print_skipped(all_tests)
# Run the tests
integration = integration_tests(
all_tests) if "integration" in test_types_to_run else 0
stress = stress_test() if "stress" in test_types_to_run else 0
unit = unit_tests() if "unit" in test_types_to_run else 0
examples = examples_working() if "examples" in test_types_to_run else 0
status = max(integration, stress, unit, examples)
if (status == 0):
print pretty.SUCCESS(
str(test_count - status) + " / " + str(test_count) +
" tests passed, exiting with code 0")
else:
print pretty.FAIL(
str(status) + " / " + str(test_count) + " tests failed ")
sys.exit(status)
def stop(self):
# Check for sudo access, needed for qemu commands
if os.getuid() is not 0:
print color.FAIL("Call the script with sudo access")
sys.exit(1)
print color.INFO(nametag), "Stopping VM..."
self._hyper.stop()
if hasattr(self, "_timer") and self._timer:
self._timer.cancel()
return self
def unit_tests():
"""Perform unit tests"""
global test_count
test_count += 1
if ("unit" in args.skip):
print pretty.WARNING("Unit tests skipped")
return 0
print pretty.HEADER("Building and running unit tests")
build_status = Test(".", name="Unit test build", command=["make"], clean = args.clean).start().wait_status()
unit_status = Test(".", name="Unit tests", command = ["./test.lest"]).start().wait_status()
if (build_status or unit_status) and args.fail:
print pretty.FAIL("Unit tests failed")
sys.exit(max(build_status, unit_status))
return max(build_status, unit_status)
def stress_test():
"""Perform stresstest"""
global test_count
test_count += 1
if ("stress" in args.skip):
print pretty.WARNING("Stress test skipped")
return 0
if (not validate_test.validate_path("stress")):
raise Exception("Stress test failed validation")
print pretty.HEADER("Starting stress test")
stress = Test("stress", clean = args.clean).start()
if (stress and args.fail):
print pretty.FAIL("Stress test failed")
sys.exit(stress)
return 1 if stress.wait_status() else 0
def integration_tests(tests):
""" Function that runs the tests that are passed to it.
Filters out any invalid tests before running
Arguments:
tests: List containing test objects to be run
Returns:
integer: Number of tests that failed
"""
# Only run the valid tests
tests = [ x for x in tests if not x.skip_ and x.type_ == 'integration' ]
# Print info before starting to run
print pretty.HEADER("Starting " + str(len(tests)) + " integration test(s)")
for test in tests:
print pretty.INFO("Test"), "starting", test.name_
processes = []
fail_count = 0
global test_count
test_count += len(tests)
# Start running tests in parallell
for test in tests:
processes.append(test.start())
# Collect test results
print pretty.HEADER("Collecting integration test results")
for p in processes:
fail_count += 1 if p.wait_status() else 0
# Exit early if any tests failed
if fail_count and args.fail:
print pretty.FAIL(str(fail_count) + "integration tests failed")
sys.exit(fail_count)
return fail_count