def _docnode_line_workaround(self, docnode):
# lineno points to the last line of a string
endpos = docnode.lineno - 1
docstr = utils.ensure_unicode(docnode.value.s)
sourcelines = self.sourcelines
start, stop = self._docstr_line_workaround(docstr, sourcelines, endpos)
# Convert 0-based line positions to 1-based line numbers
doclineno = start + 1
doclineno_end = stop
# print('docnode = {!r}'.format(docnode))
return doclineno, doclineno_end
def _docnode_line_workaround(self, docnode):
"""
Find the start and ending line numbers of a docstring
CommandLine:
xdoctest -m xdoctest.static_analysis TopLevelVisitor._docnode_line_workaround
Example:
>>> from xdoctest.static_analysis import * # NOQA
>>> sq = chr(39) # single quote
>>> dq = chr(34) # double quote
>>> source = utils.codeblock(
'''
def func0():
{ddd} docstr0 {ddd}
def func1():
{ddd}
docstr1 {ddd}
def func2():
{ddd} docstr2
{ddd}
def func3():
{ddd}
docstr3
{ddd} # foobar
def func5():
{ddd}pathological case
{sss} # {ddd} # {sss} # {ddd} # {ddd}
def func6():
" single quoted docstr "
def func7():
r{ddd}
raw line
{ddd}
''').format(ddd=dq * 3, sss=sq * 3)
>>> self = TopLevelVisitor(source)
>>> func_nodes = self.syntax_tree().body
>>> print(utils.add_line_numbers(utils.highlight_code(source), start=1))
>>> wants = [
>>> (2, 2),
>>> (4, 5),
>>> (7, 8),
>>> (10, 12),
>>> (14, 15),
>>> (17, 17),
>>> (19, 21),
>>> ]
>>> for i, func_node in enumerate(func_nodes):
>>> docnode = func_node.body[0]
>>> got = self._docnode_line_workaround(docnode)
>>> want = wants[i]
>>> print('got = {!r}'.format(got))
>>> print('want = {!r}'.format(want))
>>> assert got == want
"""
# lineno points to the last line of a string in CPython < 3.8
if hasattr(docnode, 'end_lineno'):
endpos = docnode.end_lineno - 1
else:
if PLAT_IMPL == 'PyPy':
startpos = docnode.lineno - 1
docstr = utils.ensure_unicode(docnode.value.s)
sourcelines = self.sourcelines
start, stop = self._find_docstr_endpos_workaround(
docstr, sourcelines, startpos)
# Convert 0-based line positions to 1-based line numbers
doclineno = start + 1
doclineno_end = stop + 1
return doclineno, doclineno_end
else:
# Hack for older versions
# TODO: fix in pypy
endpos = docnode.lineno - 1
docstr = utils.ensure_unicode(docnode.value.s)
sourcelines = self.sourcelines
start, stop = self._find_docstr_startpos_workaround(
docstr, sourcelines, endpos)
# Convert 0-based line positions to 1-based line numbers
doclineno = start + 1
doclineno_end = stop
# print('docnode = {!r}'.format(docnode))
return doclineno, doclineno_end
def parse_docstr_examples(docstr,
callname=None,
modpath=None,
lineno=1,
style='auto',
fpath=None,
parser_kw={}):
"""
Parses doctests from a docstr and generates example objects.
The style influences which tests are found.
Args:
docstr (str): a previously extracted docstring
callname (str, default=None):
the name of the callable (e.g. function, class, or method)
that this docstring belongs to.
modpath (str | PathLike, default=None):
original module the docstring is from
lineno (int, default=1):
the line number (starting from 1) of the docstring. i.e. if you
were to go to this line number in the source file the starting
quotes of the docstr would be on this line.
style (str, default='auto'): expected doctest style, which can
be "google", "freeform", or "auto".
fpath (str | PathLike, default=None):
the file that the docstring is from (if the file was not a module,
needed for backwards compatibility)
parser_kw (dict, default={}): passed to the parser
Yields:
xdoctest.doctest_example.DocTest : parsed example
CommandLine:
python -m xdoctest.core parse_docstr_examples
Example:
>>> from xdoctest.core import *
>>> from xdoctest import utils
>>> docstr = utils.codeblock(
... '''
... >>> 1 + 1 # xdoctest: +SKIP
... 2
... >>> 2 + 2
... 4
... ''')
>>> examples = list(parse_docstr_examples(docstr, 'name', fpath='foo.txt', style='freeform'))
>>> print(len(examples))
1
>>> examples = list(parse_docstr_examples(docstr, fpath='foo.txt'))
"""
if DEBUG:
print('Parsing docstring examples for '
'callname={} in modpath={}'.format(callname, modpath))
if style == 'freeform':
parser = parse_freeform_docstr_examples
elif style == 'google':
parser = parse_google_docstr_examples
elif style == 'auto':
parser = parse_auto_docstr_examples
# TODO:
# elif style == 'numpy':
# parser = parse_numpy_docstr_examples
else:
raise KeyError('Unknown style={}. Valid styles are {}'.format(
style, DOCTEST_STYLES))
if DEBUG:
print('parser = {!r}'.format(parser))
n_parsed = 0
try:
for example in parser(docstr,
callname=callname,
modpath=modpath,
fpath=fpath,
lineno=lineno,
**parser_kw):
n_parsed += 1
yield example
except Exception as ex:
if DEBUG:
print('Caught an error when parsing')
msg = ('Cannot scrape callname={} in modpath={} line={}.\n'
'Caused by: {}\n')
# raise
msg = msg.format(callname, modpath, lineno, repr(ex))
if isinstance(ex, exceptions.DoctestParseError):
# TODO: Can we print a nicer syntax error here?
msg += '{}\n'.format(ex.string)
msg += 'Original Error: {}\n'.format(repr(ex.orig_ex))
if isinstance(ex.orig_ex, SyntaxError):
extra_help = ''
if ex.orig_ex.text:
extra_help += utils.ensure_unicode(ex.orig_ex.text)
if ex.orig_ex.offset is not None:
extra_help += ' ' * (ex.orig_ex.offset - 1) + '^'
if extra_help:
msg += '\n' + extra_help
# Always warn when something bad is happening.
# However, dont error if the docstr simply has bad syntax
print('msg = {}'.format(msg))
warnings.warn(msg)
if isinstance(ex, exceptions.MalformedDocstr):
pass
elif isinstance(ex, exceptions.DoctestParseError):
pass
else:
raise
if DEBUG:
print('Finished parsing {} examples'.format(n_parsed))
def parse(self, string, info=None):
"""
Divide the given string into examples and interleaving text.
Args:
string (str): string representing the doctest
info (dict): info about where the string came from in case of an
error
Returns:
list : a list of `DoctestPart` objects
CommandLine:
python -m xdoctest.parser DoctestParser.parse
Example:
>>> s = 'I am a dummy example with two parts'
>>> x = 10
>>> print(s)
I am a dummy example with two parts
>>> s = 'My purpose it so demonstrate how wants work here'
>>> print('The new want applies ONLY to stdout')
>>> print('given before the last want')
>>> '''
this wont hurt the test at all
even though its multiline '''
>>> y = 20
The new want applies ONLY to stdout
given before the last want
>>> # Parts from previous examples are executed in the same context
>>> print(x + y)
30
this is simply text, and doesnt apply to the previous doctest the
<BLANKLINE> directive is still in effect.
Example:
>>> from xdoctest import parser
>>> from xdoctest.docstr import docscrape_google
>>> from xdoctest import core
>>> self = parser.DoctestParser()
>>> docstr = self.parse.__doc__
>>> blocks = docscrape_google.split_google_docblocks(docstr)
>>> doclineno = self.parse.__func__.__code__.co_firstlineno
>>> key, (string, offset) = blocks[-2]
>>> self._label_docsrc_lines(string)
>>> doctest_parts = self.parse(string)
>>> # each part with a want-string needs to be broken in two
>>> assert len(doctest_parts) == 6
"""
if DEBUG > 1:
print('\n===== PARSE ====')
if sys.version_info.major == 2: # nocover
string = utils.ensure_unicode(string)
if not isinstance(string, six.string_types):
raise TypeError('Expected string but got {!r}'.format(string))
string = string.expandtabs()
# If all lines begin with the same indentation, then strip it.
min_indent = _min_indentation(string)
if min_indent > 0:
string = '\n'.join([l[min_indent:] for l in string.splitlines()])
labeled_lines = None
grouped_lines = None
all_parts = None
try:
labeled_lines = self._label_docsrc_lines(string)
grouped_lines = self._group_labeled_lines(labeled_lines)
all_parts = list(self._package_groups(grouped_lines))
except Exception as orig_ex:
if labeled_lines is None:
failpoint = '_label_docsrc_lines'
elif grouped_lines is None:
failpoint = '_group_labeled_lines'
elif all_parts is None:
failpoint = '_package_groups'
if DEBUG:
print('<FAILPOINT>')
print('!!! FAILED !!!')
print('failpoint = {!r}'.format(failpoint))
import ubelt as ub
import traceback
tb_text = traceback.format_exc()
tb_text = ub.highlight_code(tb_text)
tb_text = ub.indent(tb_text)
print(tb_text)
print('Failed to parse string = <{[<{[<{[')
print(string)
print(']}>a]}>]}> # end string')
print('info = {}'.format(ub.repr2(info)))
print('-----')
print('orig_ex = {}'.format(orig_ex))
print('labeled_lines = {}'.format(ub.repr2(labeled_lines)))
print('grouped_lines = {}'.format(ub.repr2(grouped_lines,
nl=3)))
print('all_parts = {}'.format(ub.repr2(all_parts)))
print('</FAILPOINT>')
# sys.exit(1)
raise exceptions.DoctestParseError(
'Failed to parse doctest in {}'.format(failpoint),
string=string,
info=info,
orig_ex=orig_ex)
if DEBUG > 1:
print('\n===== FINISHED PARSE ====')
return all_parts
def output_difference(self, runstate=None, colored=True):
"""
Return a string describing the differences between the expected output
for a given example (`example`) and the actual output (`got`).
The `runstate` contains option flags used to compare `want` and `got`.
Notes:
This does not check if got matches want, it only outputs the raw
differences. Got/Want normalization may make the differences appear
more exagerated than they are.
"""
got = self.got
want = self.want
if runstate is None:
runstate = directive.RuntimeState()
# Don't normalize because it usually removes the newlines
runstate_ = runstate.to_dict()
# Don't normalize whitespaces in report for better visibility
runstate_['NORMALIZE_WHITESPACE'] = False
runstate_['IGNORE_WHITESPACE'] = False
got, want = normalize(got, want, runstate_)
# If <BLANKLINE>s are being used, then replace blank lines
# with <BLANKLINE> in the actual output string.
# if not runstate['DONT_ACCEPT_BLANKLINE']:
# got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
got = utils.ensure_unicode(got)
# Check if we should use diff.
if self._do_a_fancy_diff(runstate):
# Split want & got into lines.
want_lines = want.splitlines(True)
got_lines = got.splitlines(True)
# Use difflib to find their differences.
if runstate['REPORT_UDIFF']:
diff = difflib.unified_diff(want_lines, got_lines, n=2)
diff = list(diff)[2:] # strip the diff header
kind = 'unified diff with -expected +actual'
elif runstate['REPORT_CDIFF']:
diff = difflib.context_diff(want_lines, got_lines, n=2)
diff = list(diff)[2:] # strip the diff header
kind = 'context diff with expected followed by actual'
elif runstate['REPORT_NDIFF']:
# TODO: Is there a way to make Differ ignore whitespace if that
# runtime directive is specified?
engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
diff = list(engine.compare(want_lines, got_lines))
kind = 'ndiff with -expected +actual'
else:
raise ValueError('Invalid difflib option')
# Remove trailing whitespace on diff output.
diff = [line.rstrip() + '\n' for line in diff]
diff_text = ''.join(diff)
if colored:
diff_text = utils.highlight_code(diff_text, lexer_name='diff')
text = 'Differences (%s):\n' % kind + utils.indent(diff_text)
else:
# If we're not using diff, then simply list the expected
# output followed by the actual output.
if want and got:
if colored:
got = utils.color_text(got, 'red')
want = utils.color_text(want, 'red')
text = 'Expected:\n{}\nGot:\n{}'.format(
utils.indent(self.want), utils.indent(self.got))
elif want:
if colored:
got = utils.color_text(got, 'red')
want = utils.color_text(want, 'red')
text = 'Expected:\n{}\nGot nothing\n'.format(
utils.indent(want))
elif got: # nocover
raise AssertionError('impossible state')
text = 'Expected nothing\nGot:\n{}'.format(utils.indent(got))
else: # nocover
raise AssertionError('impossible state')
text = 'Expected nothing\nGot nothing\n'
return text
def test_runner_syntax_error():
"""
python testing/test_errors.py test_runner_syntax_error
"""
source = utils.codeblock('''
def test_parsetime_syntax_error1():
"""
Example:
>>> from __future__ import print_function
>>> print 'Parse-Time Syntax Error'
"""
def test_parsetime_syntax_error2():
"""
Example:
>>> def bad_syntax() return for
"""
def test_runtime_error():
"""
Example:
>>> print('Runtime Error {}'.format(5 / 0))
"""
def test_runtime_name_error():
"""
Example:
>>> print('Name Error {}'.format(foo))
"""
def test_runtime_warning():
"""
Example:
>>> import warnings
>>> warnings.warn('in-code warning')
"""
''')
temp = utils.TempDir(persist=True)
temp.ensure()
dpath = temp.dpath
modpath = join(dpath, 'test_runner_syntax_error.py')
open(modpath, 'w').write(source)
with utils.CaptureStdout() as cap:
runner.doctest_module(modpath,
'all',
argv=[''],
style='freeform',
verbose=0)
print('CAPTURED [[[[[[[[')
print(utils.indent(cap.text))
print(']]]]]]]] # CAPTURED')
if six.PY2:
captext = utils.ensure_unicode(cap.text)
else:
captext = cap.text
if True or not six.PY2: # Why does this have issues on the dashboards?
assert '1 run-time warnings' in captext
assert '2 parse-time warnings' in captext
# Assert summary line
assert '3 warnings' in captext
assert '2 failed' in captext
assert '1 passed' in captext
def parse(self, string, info=None):
"""
Divide the given string into examples and intervening text.
Args:
string (str): string representing the doctest
info (dict): info about where the string came from in case of an
error
Returns:
list : a list of `DoctestPart` objects
CommandLine:
python -m xdoctest.parser DoctestParser.parse
Example:
>>> s = 'I am a dummy example with two parts'
>>> x = 10
>>> print(s)
I am a dummy example with two parts
>>> s = 'My purpose it so demonstrate how wants work here'
>>> print('The new want applies ONLY to stdout')
>>> print('given before the last want')
>>> '''
this wont hurt the test at all
even though its multiline '''
>>> y = 20
The new want applies ONLY to stdout
given before the last want
>>> # Parts from previous examples are executed in the same context
>>> print(x + y)
30
this is simply text, and doesnt apply to the previous doctest the
<BLANKLINE> directive is still in effect.
Example:
>>> from xdoctest import parser
>>> from xdoctest.docstr import docscrape_google
>>> from xdoctest import core
>>> self = parser.DoctestParser()
>>> docstr = self.parse.__doc__
>>> blocks = docscrape_google.split_google_docblocks(docstr)
>>> doclineno = self.parse.__func__.__code__.co_firstlineno
>>> key, (string, offset) = blocks[-2]
>>> self._label_docsrc_lines(string)
>>> doctest_parts = self.parse(string)
>>> # each part with a want-string needs to be broken in two
>>> assert len(doctest_parts) == 6
"""
if sys.version_info.major == 2: # nocover
string = utils.ensure_unicode(string)
if not isinstance(string, six.string_types):
raise TypeError('Expected string but got {!r}'.format(string))
string = string.expandtabs()
# If all lines begin with the same indentation, then strip it.
min_indent = min_indentation(string)
if min_indent > 0:
string = '\n'.join([l[min_indent:] for l in string.splitlines()])
try:
labeled_lines = self._label_docsrc_lines(string)
grouped_lines = self._group_labeled_lines(labeled_lines)
all_parts = list(self._package_groups(grouped_lines))
except Exception as orig_ex:
# print('Failed to parse string=...')
# print(string)
# print(info)
raise exceptions.DoctestParseError('Failed to parse doctest',
string=string,
info=info,
orig_ex=orig_ex)
return all_parts