def test_run_zero_arg(): """ pytest testing/test_runner.py::test_run_zero_arg -s """ from xdoctest import runner source = utils.codeblock( ''' def zero_arg_print(): print('running zero arg') ''') with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_run_zero_arg.py') with open(modpath, 'w') as file: file.write(source) # disabled tests dont run in "all" mode with utils.CaptureStdout() as cap: try: runner.doctest_module(modpath, 'all', argv=[''], verbose=3) except Exception: pass assert 'running zero arg' not in cap.text with utils.CaptureStdout() as cap: try: runner.doctest_module(modpath, 'zero_arg_print', argv=[''], verbose=3) except Exception: pass # print(cap.text) assert 'running zero arg' in cap.text
def test_list(): from xdoctest import runner source = utils.codeblock( ''' # --- HELPERS --- def real_test1(a=1): """ Example: >>> pass """ pass def fake_test1(a=1): pass def real_test2(): """ Example: >>> pass """ pass def fake_test2(): pass ''') with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_list.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'list', argv=['']) assert 'real_test1' in cap.text assert 'real_test2' in cap.text assert 'fake_test1' not in cap.text assert 'fake_test2' not in cap.text # test command=None with utils.CaptureStdout() as cap: runner.doctest_module(modpath, None, argv=['']) assert 'real_test1' in cap.text assert 'real_test2' in cap.text assert 'fake_test1' not in cap.text assert 'fake_test2' not in cap.text
def test_runner_config(): """ pytest testing/test_runner.py::test_runner_config -s """ from xdoctest import runner source = utils.codeblock(''' def foo(): """ Example: >>> print('i wanna see this') """ ''') config = { 'default_runtime_state': { 'SKIP': True }, } with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_example_run.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'foo', argv=[''], config=config) assert 'SKIPPED' in cap.text
def _run_case(source): from xdoctest import utils COLOR = 'yellow' def cprint(msg, color=COLOR): print(utils.color_text(str(msg), COLOR)) cprint('\n\n' '\n <RUN CASE> ' '\n ======== ' '\n', COLOR) cprint('DOCTEST SOURCE:') cprint('---------------') print( utils.indent( utils.add_line_numbers(utils.highlight_code(source, 'python')))) print('') import hashlib hasher = hashlib.sha1() hasher.update(source.encode('utf8')) hashid = hasher.hexdigest()[0:8] with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_linenos_' + hashid + '.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout(supress=False) as cap: runner.doctest_module(modpath, 'all', argv=['']) cprint('\n\n --- </END RUN CASE> --- \n\n', COLOR) return cap.text
def test_global_exec(): """ pytest testing/test_runner.py::test_global_exec -s """ from xdoctest import runner source = utils.codeblock( ''' def foo(): """ Example: >>> print(a) """ ''') config = { 'global_exec': 'a=1', } with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_example_run.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'foo', argv=[''], config=config) assert '1 passed' in cap.text
def test_runner_syntax_error(): """ python testing/test_errors.py test_runner_syntax_error """ source = utils.codeblock( ''' def test_parsetime_syntax_error1(): """ Example: >>> from __future__ import print_function >>> print 'Parse-Time Syntax Error' """ def test_parsetime_syntax_error2(): """ Example: >>> def bad_syntax() return for """ def test_runtime_error(): """ Example: >>> print('Runtime Error {}'.format(5 / 0)) """ def test_runtime_name_error(): """ Example: >>> print('Name Error {}'.format(foo)) """ def test_runtime_warning(): """ Example: >>> import warnings >>> warnings.warn('in-code warning') """ ''') temp = utils.TempDir(persist=True) temp.ensure() dpath = temp.dpath modpath = join(dpath, 'test_runner_syntax_error.py') open(modpath, 'w').write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'all', argv=[''], style='freeform', verbose=0) print(utils.indent(cap.text)) assert '1 run-time warnings' in cap.text assert '2 parse-time warnings' in cap.text # Assert summary line assert '3 warnings' in cap.text assert '2 failed' in cap.text assert '1 passed' in cap.text
def test_all_disabled(): """ pytest testing/test_runner.py::test_all_disabled -s -vv python testing/test_runner.py test_all_disabled """ from xdoctest import runner source = utils.codeblock( ''' def foo(): """ Example: >>> # DISABLE_DOCTEST >>> print('all will' + ' not print this') """ def bar(): """ Example: >>> print('all will' + ' print this') """ ''') with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_all_disabled.py') with open(modpath, 'w') as file: file.write(source) # disabled tests dont run in "all" mode with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'all', argv=['']) assert 'all will print this' in cap.text # print(' ' + cap.text.replace('\n', '\n ')) assert 'all will not print this' not in cap.text # Running an disabled example explicitly should work with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'foo', argv=['']) # print(' ' + cap.text.replace('\n', '\n ')) assert 'all will not print this' in cap.text
def test_hack_the_sys_argv(): """ Tests hacky solution to issue #76 pytest testing/test_runner.py::test_global_exec -s References: https://github.com/Erotemic/xdoctest/issues/76 """ from xdoctest import runner source = utils.codeblock( ''' def foo(): """ Example: >>> # xdoctest: +REQUIRES(--hackedflag) >>> print('This will run if global_exec specified') """ ''') import sys NEEDS_FIX = '--hackedflag' not in sys.argv config = { 'global_exec': 'import sys; sys.argv.append("--hackedflag")' } with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_example_run.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'foo', argv=[''], config=config) if NEEDS_FIX: # Fix the global state sys.argv.remove('--hackedflag') # print(cap.text) assert '1 passed' in cap.text
def _run_case(source, style='auto'): """ Runs all doctests in a source block Args: source (str): source code of an entire file TODO: run case is over-duplicated and should be separated into a test utils directory """ from xdoctest import utils from xdoctest import runner COLOR = 'yellow' def cprint(msg, color=COLOR): print(utils.color_text(str(msg), COLOR)) cprint('\n\n' '\n <RUN CASE> ' '\n ======== ' '\n', COLOR) cprint('CASE SOURCE:') cprint('------------') print(utils.indent( utils.add_line_numbers(utils.highlight_code(source, 'python')))) print('') import hashlib hasher = hashlib.sha1() hasher.update(source.encode('utf8')) hashid = hasher.hexdigest()[0:8] with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_linenos_' + hashid + '.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout(supress=False) as cap: runner.doctest_module(modpath, 'all', argv=[''], style=style) cprint('\n\n --- </END RUN CASE> --- \n\n', COLOR) return cap.text
def test_example_run(): from xdoctest import runner source = utils.codeblock(''' def foo(): """ Example: >>> print('i wanna see this') """ ''') with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_example_run.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'foo', argv=['']) assert 'i wanna see this' in cap.text
def run(self, verbose=None, on_error=None): """ Executes the doctest, checks the results, reports the outcome. Args: verbose (int): verbosity level on_error (str): can be 'raise' or 'return' Returns: Dict : summary """ on_error = self.config.getvalue('on_error', on_error) verbose = self.config.getvalue('verbose', verbose) if on_error not in {'raise', 'return'}: raise KeyError(on_error) self._parse() # parse out parts if we have not already done so self._pre_run(verbose) self._import_module() # Prepare for actual test run test_globals, compileflags = self._test_globals() self.logged_evals.clear() self.logged_stdout.clear() self._unmatched_stdout = [] self._skipped_parts = [] self.exc_info = None self._suppressed_stdout = verbose <= 1 # Initialize a new runtime state default_state = self.config['default_runtime_state'] runstate = self._runstate = directive.RuntimeState(default_state) # setup reporting choice runstate.set_report_style(self.config['reportchoice'].lower()) global_exec = self.config.getvalue('global_exec') if global_exec: # Hack to make it easier to specify multi-line input on the CLI global_source = utils.codeblock(global_exec.replace('\\n', '\n')) global_code = compile(global_source, mode='exec', filename='<doctest:' + self.node + ':' + 'global_exec>', flags=compileflags, dont_inherit=True) exec(global_code, test_globals) # Can't do this because we can't force execution of SCRIPTS # if self.is_disabled(): # runstate['SKIP'] = True # - [x] TODO: fix CaptureStdout so it doesn't break embedding shells # don't capture stdout for zero-arg blocks # needs_capture = self.block_type != 'zero-arg' # I think the bug that broke embedding shells is fixed, so it is now # safe to capture. If not, uncomment above lines. If this works without # issue, then remove these notes in a future version. # needs_capture = False needs_capture = True # Use the same capture object for all parts in the test cap = utils.CaptureStdout(supress=self._suppressed_stdout, enabled=needs_capture) with warnings.catch_warnings(record=True) as self.warn_list: for partx, part in enumerate(self._parts): # Extract directives and and update runtime state runstate.update(part.directives) # Handle runtime actions if runstate['SKIP'] or len(runstate['REQUIRES']) > 0: self._skipped_parts.append(part) continue # Prepare to capture stdout and evaluated values self.failed_part = part got_eval = constants.NOT_EVALED try: # Compile code, handle syntax errors # part.compile_mode can be single, exec, or eval. # Typically single is used instead of eval self._partfilename = '<doctest:' + self.node + '>' code = compile(part.source, mode=part.compile_mode, filename=self._partfilename, flags=compileflags, dont_inherit=True) except KeyboardInterrupt: # nocover raise except Exception: raise # self.exc_info = sys.exc_info() # ex_type, ex_value, tb = self.exc_info # self.failed_tb_lineno = tb.tb_lineno # if on_error == 'raise': # raise try: # Execute the doctest code try: # NOTE: For code passed to eval or exec, there is no # difference between locals and globals. Only pass in # one dict, otherwise there is weird behavior with cap: # We can execute each part using exec or eval. If # a doctest part has `compile_mode=eval` we # exepect it to return an object with a repr that # can compared to a "want" statement. # print('part.compile_mode = {!r}'.format(part.compile_mode)) if part.compile_mode == 'eval': # print('test_globals = {}'.format(sorted(test_globals.keys()))) got_eval = eval(code, test_globals) # if EVAL_MIGHT_RETURN_COROUTINE: # import types # if isinstance(got_eval, types.CoroutineType): # # In 3.9-rc (2020-mar-31) it looks like # # eval sometimes returns coroutines. I # # found no docs on this. Not sure if it # # will be mainlined, but this seems to # # fix it. # import asyncio # got_eval = asyncio.run(got_eval) else: exec(code, test_globals) # Record any standard output and "got_eval" produced by # this doctest_part. self.logged_evals[partx] = got_eval self.logged_stdout[partx] = cap.text except Exception: if part.want: # A failure may be expected if the traceback # matches the part's want statement. exception = sys.exc_info() traceback.format_exception_only(*exception[:2]) exc_got = traceback.format_exception_only( *exception[:2])[-1] want = part.want checker.check_exception(exc_got, want, runstate) else: raise else: """ TODO: [ ] - Delay got-want failure until the end of the doctest. Allow the rest of the code to run. If multiple errors occur, show them both. """ if part.want: got_stdout = cap.text if not runstate['IGNORE_WANT']: part.check(got_stdout, got_eval, runstate, unmatched=self._unmatched_stdout) # Clear unmatched output when a check passes self._unmatched_stdout = [] else: # If a part doesnt have a want allow its output to # be matched by the next part. self._unmatched_stdout.append(cap.text) # Handle anything that could go wrong except KeyboardInterrupt: # nocover raise except (exceptions.ExitTestException, exceptions._pytest.outcomes.Skipped): if verbose > 0: print('Test gracefully exists') break except checker.GotWantException: # When the "got", does't match the "want" self.exc_info = sys.exc_info() if on_error == 'raise': raise break except checker.ExtractGotReprException as ex: # When we fail to extract the "got" self.exc_info = sys.exc_info() if on_error == 'raise': raise ex.orig_ex break except Exception as _ex_dbg: ex_type, ex_value, tb = sys.exc_info() DEBUG = 0 if DEBUG: print('_ex_dbg = {!r}'.format(_ex_dbg)) print('<DEBUG: doctest encountered exception>', file=sys.stderr) print(''.join(traceback.format_tb(tb)), file=sys.stderr) print('</DEBUG>', file=sys.stderr) # Search for the traceback that corresponds with the # doctest, and remove the parts that point to # boilerplate lines in this file. found_lineno = None for sub_tb in _traverse_traceback(tb): tb_filename = sub_tb.tb_frame.f_code.co_filename if tb_filename == self._partfilename: # Walk up the traceback until we find the one that has # the doctest as the base filename found_lineno = sub_tb.tb_lineno break if DEBUG: # The only traceback remaining should be # the part that is relevant to the user print('<DEBUG: best sub_tb>', file=sys.stderr) print('found_lineno = {!r}'.format(found_lineno), file=sys.stderr) print(''.join(traceback.format_tb(sub_tb)), file=sys.stderr) print('</DEBUG>', file=sys.stderr) if found_lineno is None: if DEBUG: print( 'UNABLE TO CLEAN TRACEBACK. EXIT DUE TO DEBUG') sys.exit(1) raise ValueError( 'Could not clean traceback: ex = {!r}'.format( _ex_dbg)) else: self.failed_tb_lineno = found_lineno self.exc_info = (ex_type, ex_value, tb) # The idea of CLEAN_TRACEBACK is to make it so the # traceback from this function doesn't clutter the error # message the user sees. if on_error == 'raise': raise break finally: if cap.enabled: assert cap.text is not None # Ensure that we logged the output even in failure cases self.logged_evals[partx] = got_eval self.logged_stdout[partx] = cap.text if self.exc_info is None: self.failed_part = None if len(self._skipped_parts) == len(self._parts): # we skipped everything if self.mode == 'pytest': import pytest pytest.skip() summary = self._post_run(verbose) # Clear the global namespace so doctests don't leak memory self.global_namespace.clear() return summary
def test_runner_failures(): """ python testing/test_runner.py test_runner_failures pytest testing/test_runner.py::test_runner_failures -s pytest testing/test_runner.py::test_all_disabled -s """ from xdoctest import runner source = utils.codeblock(''' def test1(): """ Example: >>> pass """ def test2(): """ Example: >>> assert False, 'test 2.1' Example: >>> assert False, 'test 2.2' """ def test3(): """ Example: >>> pass Example: >>> pass """ def test4(): """ Example: >>> assert False, 'test 3' """ ''') temp = utils.TempDir() temp.ensure() # with utils.TempDir() as temp: dpath = temp.dpath modpath = join(dpath, 'test_runner_failures.py') with open(modpath, 'w') as file: file.write(source) # disabled tests dont run in "all" mode with utils.CaptureStdout(supress=True) as cap: try: runner.doctest_module(modpath, 'all', argv=[''], verbose=1) except Exception: pass print('\nNOTE: the following output is part of a test') print(utils.indent(cap.text, '... ')) print('NOTE: above output is part of a test') # assert '.FFF' in cap.text assert '3 / 6 passed' in cap.text assert '3 failed 3 passed' in cap.text
def test_runner_syntax_error(): """ python testing/test_errors.py test_runner_syntax_error xdoctest -m testing/test_errors.py test_runner_syntax_error """ source = utils.codeblock(r''' def demo_parsetime_syntax_error1(): """ Example: >>> from __future__ import print_function >>> print 'Parse-Time Syntax Error' """ def demo_parsetime_syntax_error2(): """ Example: >>> def bad_syntax() return for """ def demo_runtime_error(): """ Example: >>> print('Runtime Error {}'.format(5 / 0)) """ def demo_runtime_name_error(): """ Example: >>> print('Name Error {}'.format(foo)) """ def demo_runtime_warning(): """ Example: >>> import warnings >>> warnings.warn('in-code warning') """ ''') temp = utils.TempDir(persist=True) temp.ensure() dpath = temp.dpath modpath = join(dpath, 'demo_runner_syntax_error.py') with open(modpath, 'w') as file: file.write(source) with utils.CaptureStdout() as cap: runner.doctest_module(modpath, 'all', argv=[''], style='freeform', verbose=1) print('CAPTURED [[[[[[[[') print(utils.indent(cap.text)) print(']]]]]]]] # CAPTURED') if six.PY2: captext = utils.ensure_unicode(cap.text) else: captext = cap.text if True or not six.PY2: # Why does this have issues on the dashboards? assert '1 run-time warnings' in captext assert '2 parse-time warnings' in captext # Assert summary line assert '3 warnings' in captext assert '2 failed' in captext assert '1 passed' in captext