def boot(self, timeout = None, multiboot = True, kernel_args = "booted with vmrunner"): # Check for sudo access, needed for tap network devices and the KVM module if os.getuid() is not 0: print color.FAIL("Call the script with sudo access") sys.exit(1) # Start the timeout thread if (timeout): self._timer = threading.Timer(timeout, self._on_timeout) self._timer.start() # Boot via hypervisor try: self._hyper.boot(multiboot, kernel_args + "/" + self._hyper.name()) except Exception as err: print color.WARNING("Exception raised while booting ") if (timeout): self._timer.cancel() raise err # Start analyzing output while self._hyper.poll() == None and not self._exit_status: line = self._hyper.readline() print color.SUBPROC("<VM> "+line.rstrip()) # Look for event-triggers for pattern, func in self._on_output.iteritems(): if re.search(pattern, line): try: res = func() except Exception as err: print color.WARNING("Exception raised in event callback: ") print_exception() res = False self.stop().wait() #NOTE: It can be 'None' without problem if res == False: self._exit_status = exit_codes["OUTSIDE_FAIL"] self.exit(self._exit_status, color.FAIL(nametag + " VM-external test failed")) # Now we either have an exit status from timer thread, or an exit status # from the subprocess, or the VM was powered off by the external test. # If the process didn't exit we need to stop it. if (self.poll() == None): self.stop() self.wait() if self._exit_status: print color.WARNING(nametag + "Found non-zero exit status but process didn't end. ") print color.FAIL(nametag + "Tests failed or program error") print color.INFO(nametag),"Done running VM. Exit status: ", self._exit_status sys.exit(self._exit_status) else: print color.SUCCESS(nametag + " VM exited with 0 exit status.") print color.INFO(nametag), "Subprocess finished. Exiting with ", self._hyper.poll() sys.exit(self._hyper.poll()) raise Exception("Unexpected termination")
def __init__(self, config, hyper = qemu): self._exit_status = 0 self._config = config self._on_success = lambda : self.exit(exit_codes["SUCCESS"], color.SUCCESS(nametag + " All tests passed")) self._on_panic = lambda : self.exit(exit_codes["VM_FAIL"], color.FAIL(nametag + self._hyper.readline())) self._on_timeout = lambda : self.exit(exit_codes["TIMEOUT"], color.FAIL(nametag + " Test timed out")) self._on_output = { "PANIC" : self._on_panic, "SUCCESS" : self._on_success } assert(issubclass(hyper, hypervisor)) self._hyper = hyper(config) self._timer = None self._on_exit_success = lambda : None self._on_exit = lambda : None
def main(): global test_count # Warned about skipped tests # @note : doesn't warn if you use '-t a b c ...' to run specific tests if args.skip: for skip in args.skip: print_skipped(skip, "marked skipped on command line") test_categories = ["integration", "examples", "unit", "stress"] if args.tests: test_categories = [x for x in test_categories if x in args.tests or x == "integration"] if args.skip: test_categories = [x for x in test_categories if not x in args.skip] integration = integration_tests() if "integration" in test_categories else 0 stress = stress_test() if "stress" in test_categories else 0 unit = unit_tests() if "unit" in test_categories else 0 examples = examples_working() if "examples" in test_categories else 0 status = max(integration, stress, unit, examples) if (not test_count): print "No tests selected" exit(0) if (status == 0): print pretty.SUCCESS(str(test_count - status) + " / " + str(test_count) + " tests passed, exiting with code 0") else: print pretty.FAIL(str(status) + " / " + str(test_count) + " tests failed ") sys.exit(status)
def integration_tests(): """ Loops over all valid tests as defined by ./validate_all.py. Runs them one by one and gives an update of the statuses at the end. """ global test_count valid = valid_tests() if not valid: print pretty.WARNING("Integration tests skipped") return 0 test_count += len(valid) print pretty.HEADER("Starting " + str(len(valid)) + " integration test(s)") processes = [] fail_count = 0 for path in valid: processes.append(Test(path, clean = args.clean).start()) # Collect test results print pretty.HEADER("Collecting integration test results") for p in processes: fail_count += 1 if p.wait_status() else 0 # Exit early if any tests failed if fail_count and args.fail: print pretty.FAIL(str(fail_count) + "integration tests failed") sys.exit(fail_count) return fail_count
def main(): # Find leaf nodes leaves = find_leaf_nodes() # Populate test objects all_tests = [Test(path) for path in leaves] # Figure out which tests are to be run test_categories_to_run = [] test_types_to_run = [] if args.tests: for argument in args.tests: if argument in test_categories and argument not in args.skip: test_categories_to_run.append(argument) elif argument in test_types and argument not in args.skip: test_types_to_run.append(argument) else: print 'Test specified is not recognised, exiting' sys.exit(1) else: test_types_to_run = test_types if test_categories_to_run: # This means that a specific category has been requested specific_tests = [ test for test in all_tests if test.category_ in test_categories_to_run ] # Print which tests are skipped print_skipped(specific_tests) # Run the tests integration = integration_tests(specific_tests) else: # Print which tests are skipped print_skipped(all_tests) # Run the tests integration = integration_tests( all_tests) if "integration" in test_types_to_run else 0 stress = stress_test() if "stress" in test_types_to_run else 0 unit = unit_tests() if "unit" in test_types_to_run else 0 examples = examples_working() if "examples" in test_types_to_run else 0 status = max(integration, stress, unit, examples) if (status == 0): print pretty.SUCCESS( str(test_count - status) + " / " + str(test_count) + " tests passed, exiting with code 0") else: print pretty.FAIL( str(status) + " / " + str(test_count) + " tests failed ") sys.exit(status)
def stop(self): # Check for sudo access, needed for qemu commands if os.getuid() is not 0: print color.FAIL("Call the script with sudo access") sys.exit(1) print color.INFO(nametag), "Stopping VM..." self._hyper.stop() if hasattr(self, "_timer") and self._timer: self._timer.cancel() return self
def unit_tests(): """Perform unit tests""" global test_count test_count += 1 if ("unit" in args.skip): print pretty.WARNING("Unit tests skipped") return 0 print pretty.HEADER("Building and running unit tests") build_status = Test(".", name="Unit test build", command=["make"], clean = args.clean).start().wait_status() unit_status = Test(".", name="Unit tests", command = ["./test.lest"]).start().wait_status() if (build_status or unit_status) and args.fail: print pretty.FAIL("Unit tests failed") sys.exit(max(build_status, unit_status)) return max(build_status, unit_status)
def stress_test(): """Perform stresstest""" global test_count test_count += 1 if ("stress" in args.skip): print pretty.WARNING("Stress test skipped") return 0 if (not validate_test.validate_path("stress")): raise Exception("Stress test failed validation") print pretty.HEADER("Starting stress test") stress = Test("stress", clean = args.clean).start() if (stress and args.fail): print pretty.FAIL("Stress test failed") sys.exit(stress) return 1 if stress.wait_status() else 0
def integration_tests(tests): """ Function that runs the tests that are passed to it. Filters out any invalid tests before running Arguments: tests: List containing test objects to be run Returns: integer: Number of tests that failed """ # Only run the valid tests tests = [ x for x in tests if not x.skip_ and x.type_ == 'integration' ] # Print info before starting to run print pretty.HEADER("Starting " + str(len(tests)) + " integration test(s)") for test in tests: print pretty.INFO("Test"), "starting", test.name_ processes = [] fail_count = 0 global test_count test_count += len(tests) # Start running tests in parallell for test in tests: processes.append(test.start()) # Collect test results print pretty.HEADER("Collecting integration test results") for p in processes: fail_count += 1 if p.wait_status() else 0 # Exit early if any tests failed if fail_count and args.fail: print pretty.FAIL(str(fail_count) + "integration tests failed") sys.exit(fail_count) return fail_count