def test_example(root, filename): file_path = os.path.join(root, filename) if ON_WINDOWS and any( [file_path.endswith(skip_name) for skip_name in SKIP_ON_WINDOWS]): pytest.skip() elif any([file_path.endswith(skip_name) for skip_name in SKIP]): pytest.skip() with change_directory(root), open(filename) as file_obj: file_obj.readline() second_line = file_obj.readline() try: subprocess.check_output([sys.executable, filename], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: out = e.output.decode() for exception in KNOWN_EXCEPTIONS: if re.search(exception, out): pytest.xfail() assert ("Exception in test_plan definition" not in out), "Exception raised in test_plan definition." assert ( "Traceback (most recent call last):" not in out), "Exception raised during test:\n{}".format(out) assert ("# This plan contains tests that demonstrate failures " "as well.") == second_line.strip( ), "Expected '{}' example to pass, it failed.\n{}".format( file_path, out)
def run_example_in_process(filename, root, known_exceptions, cmdline_args=None): sys_path = copy.copy(sys.path) sys_argv = copy.copy(sys.argv) cmdline_args = cmdline_args or [] file_path = os.path.join(root, filename) second_line = "" try: # set up like an external invocation. # add current dir to front of sys.path # no arguments pass in command line sys.path.insert(0, root) sys.argv = [""] + cmdline_args with change_directory(root), open(filename) as file_obj: file_obj.readline() second_line = file_obj.readline().strip() runpy.run_path(os.path.join(root, filename), run_name="__main__") except SystemExit as e: if e.code not in SUCCES_EXIT_CODES: assert ( "# This plan contains tests that demonstrate failures as well." ) == second_line, ( 'Expected "{}" example to pass, it failed'.format(file_path)) except Exception as e: for exception in known_exceptions: if re.search(exception, "{}: {}".format(type(e).__name__, str(e))): pytest.xfail() pytest.fail(traceback.format_exc()) finally: # clean up after execution # remove the modules imported from the directory of testplan_path # remove the added root from sys.path # reset the args as it was before execution for mod_name in get_local_modules(root): del sys.modules[mod_name] sys.path = sys_path sys.argv = sys_argv