def run_single_test(config, tutorial, timeout): """ Builds and runs the solution to a given tutorial application for a given configuration, checking that the result matches the completion text """ try: completion_text = COMPLETION[tutorial] except KeyError: logging.error("No completion text provided for %s." % tutorial) sys.exit(1) # Create temporary directory for working in (make this a common helper to share with init.py) build_dir = tempfile.mkdtemp(dir=TOP_LEVEL_DIR, prefix='build_') # Initialize build directory (check results?) result = common.init_build_directory(config, tutorial, True, build_dir, output=sys.stdout) if result.exit_code != 0: logging.error( "Failed to initialize build directory. Not deleting build directory %s" % build_dir) sys.exit(1) # Build result = sh.ninja(_out=sys.stdout, _cwd=build_dir) if result.exit_code != 0: logging.error("Failed to build. Not deleting build directory %s" % build_dir) sys.exit(1) # run the test, storting output in a temporary file temp_file = tempfile.NamedTemporaryFile(delete=True) logging.info("Running ./simulate") test = pexpect.spawn("python", args=["simulate"], cwd=build_dir) test.logfile = temp_file expect_strings = [completion_text] + FAILURE_TEXTS result = test.expect(expect_strings, timeout=timeout) # result is the index in the completion text list corresponding to the # text that was produced if result == 0: logging.info("Success!") else: print("<failure type='failure'>") # print the log file's contents to help debug the failure temp_file.seek(0) print(xml.sax.saxutils.escape(temp_file.read())) print_pexpect_failure(expect_strings[result]) print("</failure>") for proc in psutil.process_iter(): if "qemu" in proc.name(): proc.kill() temp_file.close() shutil.rmtree(build_dir)
def run_single_test_iteration(build_dir, solution, logfile): # Build result = sh.ninja(_out=logfile, _cwd=build_dir) if result.exit_code != 0: logging.error("Failed to build. Not deleting build directory %s" % build_dir) sys.exit(1) check = sh.Command(os.path.join(build_dir, "check")) if solution: result = check(_out=logfile, _cwd=build_dir) else: # We check the start state if not solution result = check("--start", _out=logfile, _cwd=build_dir) for proc in psutil.process_iter(): if "qemu" in proc.name(): proc.kill() return result.exit_code