def test_blankline_not_accept():
# Check that blankline is not normalized away when
# DONT_ACCEPT_BLANKLINE is on
runstate = directive.RuntimeState({'DONT_ACCEPT_BLANKLINE': True})
got = 'foo\n\nbar'
want = 'foo\n<BLANKLINE>\nbar'
assert not checker.check_output(got, want, runstate)
def test_blankline_accept():
"""
pytest testing/test_checker.py
"""
# Check that blankline is normalized away
runstate = directive.RuntimeState({'DONT_ACCEPT_BLANKLINE': False})
got = 'foo\n\nbar'
want = 'foo\n<BLANKLINE>\nbar'
assert checker.check_output(got, want, runstate)
def check_output(got, want, runstate=None):
"""
Does the actual comparison between `got` and `want`
"""
if not want: # nocover
return True
if want:
# Try default
if got == want:
return True
if runstate is None:
runstate = directive.RuntimeState()
got, want = normalize(got, want, runstate)
return _check_match(got, want, runstate)
return False
def _do_a_fancy_diff(self, runstate=None):
# Not unless they asked for a fancy diff.
got = self.got
want = self.want
if runstate is None:
runstate = directive.RuntimeState()
# ndiff does intraline difference marking, so can be useful even
# for 1-line differences.
if runstate['REPORT_NDIFF']:
return True
# The other diff types need at least a few lines to be helpful.
if runstate['REPORT_UDIFF'] or runstate['REPORT_CDIFF']:
return want.count('\n') > 2 and got.count('\n') > 2
return False
def output_repr_difference(self, runstate=None):
"""
Constructs a repr difference with minimal normalization.
"""
minimal_got = self.got.rstrip()
minimal_want = self.want.rstrip()
if runstate is None:
runstate = directive.RuntimeState()
# Don't normalize because it usually removes the newlines
runstate_ = runstate.to_dict()
if not runstate_['DONT_ACCEPT_BLANKLINE']:
minimal_want = remove_blankline_marker(minimal_want)
lines = [
('Repr Difference:'),
# TODO: get a semi-normalized output before showing repr?
(' got = {!r}'.format(minimal_got)),
(' want = {!r}'.format(minimal_want)),
]
return '\n'.join(lines)
def test_blankline_failcase():
# Check that blankline is not normalizd in a "got" statement
runstate = directive.RuntimeState({'DONT_ACCEPT_BLANKLINE': False})
got = 'foo\n<BLANKLINE>\nbar'
want = 'foo\n\nbar'
assert not checker.check_output(got, want, runstate)
def run(self, verbose=None, on_error=None):
"""
Executes the doctest, checks the results, reports the outcome.
Args:
verbose (int): verbosity level
on_error (str): can be 'raise' or 'return'
Returns:
Dict : summary
"""
on_error = self.config.getvalue('on_error', on_error)
verbose = self.config.getvalue('verbose', verbose)
if on_error not in {'raise', 'return'}:
raise KeyError(on_error)
self._parse() # parse out parts if we have not already done so
self._pre_run(verbose)
self._import_module()
# Prepare for actual test run
test_globals, compileflags = self._test_globals()
self.logged_evals.clear()
self.logged_stdout.clear()
self._unmatched_stdout = []
self._skipped_parts = []
self.exc_info = None
self._suppressed_stdout = verbose <= 1
# Initialize a new runtime state
default_state = self.config['default_runtime_state']
runstate = self._runstate = directive.RuntimeState(default_state)
# setup reporting choice
runstate.set_report_style(self.config['reportchoice'].lower())
global_exec = self.config.getvalue('global_exec')
if global_exec:
# Hack to make it easier to specify multi-line input on the CLI
global_source = utils.codeblock(global_exec.replace('\\n', '\n'))
global_code = compile(global_source,
mode='exec',
filename='<doctest:' + self.node + ':' +
'global_exec>',
flags=compileflags,
dont_inherit=True)
exec(global_code, test_globals)
# Can't do this because we can't force execution of SCRIPTS
# if self.is_disabled():
# runstate['SKIP'] = True
# - [x] TODO: fix CaptureStdout so it doesn't break embedding shells
# don't capture stdout for zero-arg blocks
# needs_capture = self.block_type != 'zero-arg'
# I think the bug that broke embedding shells is fixed, so it is now
# safe to capture. If not, uncomment above lines. If this works without
# issue, then remove these notes in a future version.
# needs_capture = False
needs_capture = True
# Use the same capture object for all parts in the test
cap = utils.CaptureStdout(supress=self._suppressed_stdout,
enabled=needs_capture)
with warnings.catch_warnings(record=True) as self.warn_list:
for partx, part in enumerate(self._parts):
# Extract directives and and update runtime state
runstate.update(part.directives)
# Handle runtime actions
if runstate['SKIP'] or len(runstate['REQUIRES']) > 0:
self._skipped_parts.append(part)
continue
# Prepare to capture stdout and evaluated values
self.failed_part = part
got_eval = constants.NOT_EVALED
try:
# Compile code, handle syntax errors
# part.compile_mode can be single, exec, or eval.
# Typically single is used instead of eval
self._partfilename = '<doctest:' + self.node + '>'
code = compile(part.source,
mode=part.compile_mode,
filename=self._partfilename,
flags=compileflags,
dont_inherit=True)
except KeyboardInterrupt: # nocover
raise
except Exception:
raise
# self.exc_info = sys.exc_info()
# ex_type, ex_value, tb = self.exc_info
# self.failed_tb_lineno = tb.tb_lineno
# if on_error == 'raise':
# raise
try:
# Execute the doctest code
try:
# NOTE: For code passed to eval or exec, there is no
# difference between locals and globals. Only pass in
# one dict, otherwise there is weird behavior
with cap:
# We can execute each part using exec or eval. If
# a doctest part has `compile_mode=eval` we
# exepect it to return an object with a repr that
# can compared to a "want" statement.
# print('part.compile_mode = {!r}'.format(part.compile_mode))
if part.compile_mode == 'eval':
# print('test_globals = {}'.format(sorted(test_globals.keys())))
got_eval = eval(code, test_globals)
# if EVAL_MIGHT_RETURN_COROUTINE:
# import types
# if isinstance(got_eval, types.CoroutineType):
# # In 3.9-rc (2020-mar-31) it looks like
# # eval sometimes returns coroutines. I
# # found no docs on this. Not sure if it
# # will be mainlined, but this seems to
# # fix it.
# import asyncio
# got_eval = asyncio.run(got_eval)
else:
exec(code, test_globals)
# Record any standard output and "got_eval" produced by
# this doctest_part.
self.logged_evals[partx] = got_eval
self.logged_stdout[partx] = cap.text
except Exception:
if part.want:
# A failure may be expected if the traceback
# matches the part's want statement.
exception = sys.exc_info()
traceback.format_exception_only(*exception[:2])
exc_got = traceback.format_exception_only(
*exception[:2])[-1]
want = part.want
checker.check_exception(exc_got, want, runstate)
else:
raise
else:
"""
TODO:
[ ] - Delay got-want failure until the end of the
doctest. Allow the rest of the code to run. If
multiple errors occur, show them both.
"""
if part.want:
got_stdout = cap.text
if not runstate['IGNORE_WANT']:
part.check(got_stdout,
got_eval,
runstate,
unmatched=self._unmatched_stdout)
# Clear unmatched output when a check passes
self._unmatched_stdout = []
else:
# If a part doesnt have a want allow its output to
# be matched by the next part.
self._unmatched_stdout.append(cap.text)
# Handle anything that could go wrong
except KeyboardInterrupt: # nocover
raise
except (exceptions.ExitTestException,
exceptions._pytest.outcomes.Skipped):
if verbose > 0:
print('Test gracefully exists')
break
except checker.GotWantException:
# When the "got", does't match the "want"
self.exc_info = sys.exc_info()
if on_error == 'raise':
raise
break
except checker.ExtractGotReprException as ex:
# When we fail to extract the "got"
self.exc_info = sys.exc_info()
if on_error == 'raise':
raise ex.orig_ex
break
except Exception as _ex_dbg:
ex_type, ex_value, tb = sys.exc_info()
DEBUG = 0
if DEBUG:
print('_ex_dbg = {!r}'.format(_ex_dbg))
print('<DEBUG: doctest encountered exception>',
file=sys.stderr)
print(''.join(traceback.format_tb(tb)),
file=sys.stderr)
print('</DEBUG>', file=sys.stderr)
# Search for the traceback that corresponds with the
# doctest, and remove the parts that point to
# boilerplate lines in this file.
found_lineno = None
for sub_tb in _traverse_traceback(tb):
tb_filename = sub_tb.tb_frame.f_code.co_filename
if tb_filename == self._partfilename:
# Walk up the traceback until we find the one that has
# the doctest as the base filename
found_lineno = sub_tb.tb_lineno
break
if DEBUG:
# The only traceback remaining should be
# the part that is relevant to the user
print('<DEBUG: best sub_tb>', file=sys.stderr)
print('found_lineno = {!r}'.format(found_lineno),
file=sys.stderr)
print(''.join(traceback.format_tb(sub_tb)),
file=sys.stderr)
print('</DEBUG>', file=sys.stderr)
if found_lineno is None:
if DEBUG:
print(
'UNABLE TO CLEAN TRACEBACK. EXIT DUE TO DEBUG')
sys.exit(1)
raise ValueError(
'Could not clean traceback: ex = {!r}'.format(
_ex_dbg))
else:
self.failed_tb_lineno = found_lineno
self.exc_info = (ex_type, ex_value, tb)
# The idea of CLEAN_TRACEBACK is to make it so the
# traceback from this function doesn't clutter the error
# message the user sees.
if on_error == 'raise':
raise
break
finally:
if cap.enabled:
assert cap.text is not None
# Ensure that we logged the output even in failure cases
self.logged_evals[partx] = got_eval
self.logged_stdout[partx] = cap.text
if self.exc_info is None:
self.failed_part = None
if len(self._skipped_parts) == len(self._parts):
# we skipped everything
if self.mode == 'pytest':
import pytest
pytest.skip()
summary = self._post_run(verbose)
# Clear the global namespace so doctests don't leak memory
self.global_namespace.clear()
return summary
def output_difference(self, runstate=None, colored=True):
"""
Return a string describing the differences between the expected output
for a given example (`example`) and the actual output (`got`).
The `runstate` contains option flags used to compare `want` and `got`.
Notes:
This does not check if got matches want, it only outputs the raw
differences. Got/Want normalization may make the differences appear
more exagerated than they are.
"""
got = self.got
want = self.want
if runstate is None:
runstate = directive.RuntimeState()
# Don't normalize because it usually removes the newlines
runstate_ = runstate.to_dict()
# Don't normalize whitespaces in report for better visibility
runstate_['NORMALIZE_WHITESPACE'] = False
runstate_['IGNORE_WHITESPACE'] = False
got, want = normalize(got, want, runstate_)
# If <BLANKLINE>s are being used, then replace blank lines
# with <BLANKLINE> in the actual output string.
# if not runstate['DONT_ACCEPT_BLANKLINE']:
# got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
got = utils.ensure_unicode(got)
# Check if we should use diff.
if self._do_a_fancy_diff(runstate):
# Split want & got into lines.
want_lines = want.splitlines(True)
got_lines = got.splitlines(True)
# Use difflib to find their differences.
if runstate['REPORT_UDIFF']:
diff = difflib.unified_diff(want_lines, got_lines, n=2)
diff = list(diff)[2:] # strip the diff header
kind = 'unified diff with -expected +actual'
elif runstate['REPORT_CDIFF']:
diff = difflib.context_diff(want_lines, got_lines, n=2)
diff = list(diff)[2:] # strip the diff header
kind = 'context diff with expected followed by actual'
elif runstate['REPORT_NDIFF']:
# TODO: Is there a way to make Differ ignore whitespace if that
# runtime directive is specified?
engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
diff = list(engine.compare(want_lines, got_lines))
kind = 'ndiff with -expected +actual'
else:
raise ValueError('Invalid difflib option')
# Remove trailing whitespace on diff output.
diff = [line.rstrip() + '\n' for line in diff]
diff_text = ''.join(diff)
if colored:
diff_text = utils.highlight_code(diff_text, lexer_name='diff')
text = 'Differences (%s):\n' % kind + utils.indent(diff_text)
else:
# If we're not using diff, then simply list the expected
# output followed by the actual output.
if want and got:
if colored:
got = utils.color_text(got, 'red')
want = utils.color_text(want, 'red')
text = 'Expected:\n{}\nGot:\n{}'.format(
utils.indent(self.want), utils.indent(self.got))
elif want:
if colored:
got = utils.color_text(got, 'red')
want = utils.color_text(want, 'red')
text = 'Expected:\n{}\nGot nothing\n'.format(
utils.indent(want))
elif got: # nocover
raise AssertionError('impossible state')
text = 'Expected nothing\nGot:\n{}'.format(utils.indent(got))
else: # nocover
raise AssertionError('impossible state')
text = 'Expected nothing\nGot nothing\n'
return text
def normalize(got, want, runstate=None):
r"""
Adapated from doctest_nose_plugin.py from the nltk project:
https://github.com/nltk/nltk
Further extended to also support byte literals.
Example:
>>> want = "...\n(0, 2, {'weight': 1})\n(0, 3, {'weight': 2})"
>>> got = "(0, 2, {'weight': 1})\n(0, 3, {'weight': 2})"
"""
if runstate is None:
runstate = directive.RuntimeState()
def remove_prefixes(regex, text):
return re.sub(regex, r'\1\2', text)
def visible_text(lines):
# TODO: backspaces
# Any lines that end with only a carrage return are erased
return [line for line in lines if not line.endswith('\r')]
# Remove terminal colors
if True:
got = utils.strip_ansi(got)
want = utils.strip_ansi(want)
if True:
# normalize python 2/3 byte/unicode prefixes
got = remove_prefixes(unicode_literal_re, got)
want = remove_prefixes(unicode_literal_re, want)
# Note: normalizing away prefixes can cause weird "got"
# results to print when there is a got-want mismatch.
# For instance, if you get {'b': 22} but you want {'b': 2}
# this will cause xdoctest to report that you wanted {'': 2}
# because it reports the normalized version of the want message
got = remove_prefixes(bytes_literal_re, got)
want = remove_prefixes(bytes_literal_re, want)
# Replace <BLANKLINE>s if it is being used.
if not runstate['DONT_ACCEPT_BLANKLINE']:
want = remove_blankline_marker(want)
# always remove trailing whitepsace
got = re.sub(TRAILING_WS, '', got)
want = re.sub(TRAILING_WS, '', want)
# normalize endling newlines
want = want.rstrip()
got = got.rstrip()
# Always remove invisible text
got_lines = got.splitlines(True)
want_lines = want.splitlines(True)
got_lines = visible_text(got_lines)
want_lines = visible_text(want_lines)
want = ''.join(want_lines)
got = ''.join(got_lines)
if runstate['NORMALIZE_WHITESPACE'] or runstate['IGNORE_WHITESPACE']:
# all whitespace normalization
# treat newlines and all whitespace as a single space
got = ' '.join(got.split())
want = ' '.join(want.split())
if runstate['IGNORE_WHITESPACE']:
# Completely remove whitespace
got = re.sub(r'\s', '', got, flags=re.MULTILINE)
want = re.sub(r'\s', '', want, flags=re.MULTILINE)
if runstate['NORMALIZE_REPR']:
def norm_repr(a, b):
# If removing quotes would allow for a match, remove them.
if not _check_match(a, b, runstate):
for q in ['"', "'"]:
if a.startswith(q) and a.endswith(q):
if _check_match(a[1:-1], b, runstate):
return a[1:-1]
return a
got = norm_repr(got, want)
want = norm_repr(want, got)
return got, want
