def test_example(root, filename):
file_path = os.path.join(root, filename)
if ON_WINDOWS and any(
[file_path.endswith(skip_name) for skip_name in SKIP_ON_WINDOWS]):
pytest.skip()
elif any([file_path.endswith(skip_name) for skip_name in SKIP]):
pytest.skip()
with change_directory(root), open(filename) as file_obj:
file_obj.readline()
second_line = file_obj.readline()
try:
subprocess.check_output([sys.executable, filename],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
out = e.output.decode()
for exception in KNOWN_EXCEPTIONS:
if re.search(exception, out):
pytest.xfail()
assert ("Exception in test_plan definition"
not in out), "Exception raised in test_plan definition."
assert (
"Traceback (most recent call last):"
not in out), "Exception raised during test:\n{}".format(out)
assert ("# This plan contains tests that demonstrate failures "
"as well.") == second_line.strip(
), "Expected '{}' example to pass, it failed.\n{}".format(
file_path, out)
def run_example_in_process(filename,
root,
known_exceptions,
cmdline_args=None):
sys_path = copy.copy(sys.path)
sys_argv = copy.copy(sys.argv)
cmdline_args = cmdline_args or []
file_path = os.path.join(root, filename)
second_line = ""
try:
# set up like an external invocation.
# add current dir to front of sys.path
# no arguments pass in command line
sys.path.insert(0, root)
sys.argv = [""] + cmdline_args
with change_directory(root), open(filename) as file_obj:
file_obj.readline()
second_line = file_obj.readline().strip()
runpy.run_path(os.path.join(root, filename), run_name="__main__")
except SystemExit as e:
if e.code not in SUCCES_EXIT_CODES:
assert (
"# This plan contains tests that demonstrate failures as well."
) == second_line, (
'Expected "{}" example to pass, it failed'.format(file_path))
except Exception as e:
for exception in known_exceptions:
if re.search(exception, "{}: {}".format(type(e).__name__, str(e))):
pytest.xfail()
pytest.fail(traceback.format_exc())
finally:
# clean up after execution
# remove the modules imported from the directory of testplan_path
# remove the added root from sys.path
# reset the args as it was before execution
for mod_name in get_local_modules(root):
del sys.modules[mod_name]
sys.path = sys_path
sys.argv = sys_argv
