def testfile(filename, fileErrors): """Funció testfile() que rep per paràmetres el filename que seria el fitxer doctest que volem mostrar per pantalla, i el fitxer fileErrors que serà els links dels errors que volem mostrar. La funció crearà una instància de myDocTestRunner, i instanciarà un paràmetre strings que serà una llista d'strings els cuals l[n] i l[n+1] seran els strings en el cual el n text estarà entremig""" fitxer = open(filename, "r").read() l = DocTestParser().parse(fitxer) l1 = [] for i in l: if isinstance(i, str): l1.append(i) l2 = DocTestParser().get_examples(fitxer) mapp = dict() doc = doctest.DocTest(l2, mapp, "", None, None, None) runner = MyDocTestRunner() runner.strings = l1 runner.run(doc) print("\n.. Enllacos als errors") with open(fileErrors, "r") as f: print(f.read())
def run_doctest(name, doctest_string, global_environment): """ Run a single test with given global_environment. Returns (True, '') if the doctest passes. Returns (False, failure_message) if the doctest fails. """ examples = doctest.DocTestParser().parse(doctest_string, name) test = doctest.DocTest( [e for e in examples if isinstance(e, doctest.Example)], global_environment, name, None, None, doctest_string, ) doctestrunner = doctest.DocTestRunner(verbose=True) runresults = io.StringIO() with redirect_stdout(runresults), redirect_stderr( runresults), hide_outputs(): doctestrunner.run(test, clear_globs=False) with open("/dev/null", "w") as f, redirect_stderr(f), redirect_stdout(f): result = doctestrunner.summarize(verbose=True) # An individual test can only pass or fail if result.failed == 0: return (True, "") else: return False, runresults.getvalue()
def run_setup_cleanup(runner, testcodes, what): # type: (Any, List[TestCode], Any) -> bool examples = [] for testcode in testcodes: example = doctest.Example(testcode.code, "", lineno=testcode.lineno) examples.append(example) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest( examples, {}, "%s (%s code)" % (group.name, what), testcodes[0].filename, 0, None, ) sim_doctest.globs = ns old_f = runner.failures self.type = "exec" # the snippet may contain multiple statements runner.run(sim_doctest, out=self._warn_out, clear_globs=False) if runner.failures > old_f: return False return True
def run_tests(ipynb_path, globs): base_path = os.path.dirname(ipynb_path) test_files = glob(os.path.join(base_path, 'tests/q*.py')) tests = [] doctestparser = doctest.DocTestParser() results = [] for test_file in test_files: test_file_globals = {} with open(test_file) as f: doctestrunner = doctest.DocTestRunner() exec(f.read(), test_file_globals) defined_test = test_file_globals['test'] assert len(defined_test['suites']) == 1 assert defined_test['points'] == 1 for case in defined_test['suites'][0]['cases']: examples = doctestparser.parse( case['code'], defined_test['name'], ) test = doctest.DocTest( [e for e in examples if type(e) is doctest.Example], globs, defined_test['name'], None, None, None) with open('/dev/null', 'w') as f, redirect_stdout(f), redirect_stderr(f): doctestrunner.run(test, clear_globs=False) with open('/dev/null', 'w') as f, redirect_stdout(f), redirect_stderr(f): result = doctestrunner.summarize() results.append(1 if result.failed == 0 else 0) return (sum(results) / len(results))
def run_doctest(name, doctest_string, global_environment): """ Run a single test with given ``global_environment``. Returns ``(True, '')`` if the doctest passes. Returns ``(False, failure_message)`` if the doctest fails. Args: name (``str``): name of doctest doctest_string (``str``): doctest in string form global_environment (``dict``): global environment resulting from the execution of a python script/notebook Returns: ``tuple`` of (``bool``, ``str``): results from running the test """ examples = doctest.DocTestParser().parse(doctest_string, name) test = doctest.DocTest( [e for e in examples if isinstance(e, doctest.Example)], global_environment, name, None, None, doctest_string) doctestrunner = doctest.DocTestRunner(verbose=True) runresults = io.StringIO() with redirect_stdout(runresults), redirect_stderr( runresults), hide_outputs(): doctestrunner.run(test, clear_globs=False) with open(os.devnull, 'w') as f, redirect_stderr(f), redirect_stdout(f): result = doctestrunner.summarize(verbose=True) # An individual test can only pass or fail if result.failed == 0: return (True, '') else: return False, runresults.getvalue()
def run_setup_cleanup(runner, testcodes, what): # type: (Any, List[TestCode], Any) -> bool examples = [] for testcode in testcodes: examples.append( doctest.Example( # type: ignore doctest_encode(testcode.code, self.env.config.source_encoding), '', # type: ignore # NOQA lineno=testcode.lineno)) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest( examples, {}, # type: ignore '%s (%s code)' % (group.name, what), filename_str, 0, None) sim_doctest.globs = ns old_f = runner.failures self.type = 'exec' # the snippet may contain multiple statements runner.run(sim_doctest, out=self._warn_out, clear_globs=False) if runner.failures > old_f: return False return True
def collect(): self.ns.update(mod.__dict__) for example in examples: atest = dt.DocTest([example], self.ns, mod.__name__, mod.__file__, maybe_lineof(func), str(func.__doc__)) yield scenario(atest)
def __call__(self, global_environment): """ Run test with given global_environment. """ test = doctest.DocTest( [e for e in self.examples if isinstance(e, doctest.Example)], global_environment, self.name, None, None, self.doctest_string ) doctestrunner = doctest.DocTestRunner(verbose=True) runresults = io.StringIO() with redirect_stdout(runresults), redirect_stderr(runresults): doctestrunner.run(test, clear_globs=False) with open('/dev/null', 'w') as f, redirect_stderr(f), redirect_stdout(f): result = doctestrunner.summarize(verbose=True) # An individual test can only pass or fail if result.failed == 0: grade = 1.0 else: grade = 0.0 if grade == 1.0: summary = 'Test {} passed!'.format(self.name) else: summary = self.PLAIN_TEXT_SUMMARY_TEMPLATE.format( name=self.name, doctest_string=dedent(self.doctest_string), runresults=runresults.getvalue() ) return TestResult(grade, summary)
def __call__(self, global_environment): """ Run test with given global_environment. """ test = doctest.DocTest( [e for e in self.examples if type(e) is doctest.Example], global_environment, self.name, None, None, self.doctest_string ) doctestrunner = doctest.DocTestRunner(verbose=True) runresults = io.StringIO() with redirect_stdout(runresults), redirect_stderr(runresults): doctestrunner.run(test, clear_globs=False) with open('/dev/null', 'w') as f, redirect_stderr(f), redirect_stdout(f): result = doctestrunner.summarize(verbose=True) score = 1.0 - (result.failed / result.attempted) if score == 1.0: summary = 'Test {} passed!'.format(self.name) else: summary = self.PLAIN_TEXT_FAILURE_SUMMARY_TEMPLATE.format( name=self.name, doctest_string=dedent(self.doctest_string), runresults=runresults.getvalue() ) return TestResult(score, summary)
def get_doctest(self, string, globs, name, filename, lineno): """ Extract all doctest examples from the given string, and collect them into a `DocTest` object. `globs`, `name`, `filename`, and `lineno` are attributes for the new `DocTest` object. See the documentation for `DocTest` for more information. """ return doctest.DocTest(self.get_examples(string, name), globs, name, filename, lineno, string)
def run_setup_cleanup(runner, testcodes, what): examples = [] for testcode in testcodes: examples.append( doctest.Example(testcode.code, '', lineno=testcode.lineno)) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest(examples, {}, '%s (%s code)' % (group.name, what), filename, 0, None) old_f = runner.failures self.type = 'exec' # the snippet may contain multiple statements runner.run(sim_doctest, out=self._warn_out) if runner.failures > old_f: return False return True
def make_suite(): # pragma: no cover from calmjs.parse.lexers import es5 as es5lexer from calmjs.parse import walkers from calmjs.parse import sourcemap def open(p, flag='r'): result = StringIO(examples[p] if flag == 'r' else '') result.name = p return result parser = doctest.DocTestParser() optflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS dist = get_distribution('calmjs.parse') if dist: if dist.has_metadata('PKG-INFO'): pkgdesc = dist.get_metadata('PKG-INFO').replace('\r', '') elif dist.has_metadata('METADATA'): pkgdesc = dist.get_metadata('METADATA').replace('\r', '') else: pkgdesc = '' pkgdesc_tests = [ t for t in parser.parse(pkgdesc) if isinstance(t, doctest.Example) ] test_loader = unittest.TestLoader() test_suite = test_loader.discover('calmjs.parse.tests', pattern='test_*.py', top_level_dir=dirname(__file__)) test_suite.addTest(doctest.DocTestSuite(es5lexer, optionflags=optflags)) test_suite.addTest(doctest.DocTestSuite(walkers, optionflags=optflags)) test_suite.addTest(doctest.DocTestSuite(sourcemap, optionflags=optflags)) test_suite.addTest( doctest.DocTestCase( # skipping all the error case tests which should all be in the # troubleshooting section at the end; bump the index whenever # more failure examples are added. # also note that line number is unknown, as PKG_INFO has headers # and also the counter is somehow inaccurate in this case. doctest.DocTest(pkgdesc_tests[:-1], {'open': open}, 'PKG_INFO', 'README.rst', None, pkgdesc), optionflags=optflags, )) return test_suite
def get_doctest(self, string, globs, name, filename, lineno): try: self.javascript_remote_session.connect() self.skip_javascript_tests = False except JavascriptSessionError as e: self.skip_javascript_tests = True ex = e globs = globs.copy() globs["_js_test"] = self.javascript_remote_session.test _doctest = doctest.DocTest(self.get_examples(string, name), globs, name, filename, lineno, string) if self.skip_javascript_tests and self.has_javascript_tests: print( "[Warning] The javascript tests will BE SKIPPED! because the connection failed:\n %s" % str(ex)) return _doctest
def test_group(self, group, filename): j = Popen(["../julia"], stdin=PIPE, stdout=PIPE, stderr=STDOUT) j.stdin.write("macro raw_str(s) s end;nothing\n".encode('utf-8')) j.stdin.write("_ans = nothing\n".encode('utf-8')) j.stdin.write(""" if VERSION >= v"0.5.0-dev+1911" pushdisplay(TextDisplay( IOContext(IOContext(STDOUT, :multiline => true), :limit => true) )); nothing end """.encode('utf-8')) self.setup_runner.julia = j self.test_runner.julia = j self.cleanup_runner.julia = j def run_setup_cleanup(runner, testcodes, what): examples = [] for testcode in testcodes: examples.append( doctest.Example(testcode.code, '', lineno=testcode.lineno)) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest(examples, {}, '%s (%s code)' % (group.name, what), filename, 0, None) old_f = runner.failures self.type = 'exec' # the snippet may contain multiple statements runner.run(sim_doctest, out=self._warn_out) if runner.failures > old_f: return False return True # run the setup code if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'): # if setup failed, don't run the group return # run the tests for code in group.tests: if len(code) == 1: # ordinary doctests (code/output interleaved) try: test = parser.get_doctest(code[0].code, {}, group.name, filename, code[0].lineno) except Exception: self.warn( 'ignoring invalid doctest code: %r' % code[0].code, '%s:%s' % (filename, code[0].lineno)) raise continue if not test.examples: continue for example in test.examples: # apply directive's comparison options new_opt = code[0].options.copy() new_opt.update(example.options) example.options = new_opt self.type = 'single' # as for ordinary doctests else: # testcode and output separate output = code[1] and code[1].code or '' options = code[1] and code[1].options or {} # disable <BLANKLINE> processing as it is not needed options[doctest.DONT_ACCEPT_BLANKLINE] = True # find out if we're testing an exception m = parser._EXCEPTION_RE.match(output) if m: exc_msg = m.group('msg') else: exc_msg = None example = doctest.Example(code[0].code, output, exc_msg=exc_msg, lineno=code[0].lineno, options=options) test = doctest.DocTest([example], {}, group.name, filename, code[0].lineno, None) self.type = 'exec' # multiple statements again self.test_runner.run(test, out=self._warn_out) # run the cleanup run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup') j.kill()
def parse_rst_ipython_tests(rst, name, extraglobs=None, optionflags=None): """Extracts examples from an rst file and produce a test suite by running them through pandas to get the expected outputs. """ # Optional dependency. import IPython from traitlets.config import Config def get_indent(line): return len(line) - len(line.lstrip()) def is_example_line(line): line = line.strip() return line and not line.startswith( '#') and not line[0] == line[-1] == ':' IMPORT_PANDAS = 'import pandas as pd' example_srcs = [] lines = iter([(lineno, line.rstrip()) for lineno, line in enumerate(rst.split('\n')) if is_example_line(line)] + [(None, 'END')]) # https://ipython.readthedocs.io/en/stable/sphinxext.html lineno, line = next(lines) while True: if line == 'END': break if line.startswith('.. ipython::'): lineno, line = next(lines) indent = get_indent(line) example = [] example_srcs.append((lineno, example)) while get_indent(line) >= indent: if '@verbatim' in line or ':verbatim:' in line or '@savefig' in line: example_srcs.pop() break line = re.sub(r'In \[\d+\]: ', '', line) line = re.sub(r'\.\.\.+:', '', line) example.append(line[indent:]) lineno, line = next(lines) if get_indent(line) == indent and line[indent] not in ')]}': example = [] example_srcs.append((lineno, example)) else: lineno, line = next(lines) # TODO(robertwb): Would it be better to try and detect/compare the actual # objects in two parallel sessions than make (stringified) doctests? examples = [] config = Config() config.HistoryManager.hist_file = ':memory:' config.InteractiveShell.autocall = False config.InteractiveShell.autoindent = False config.InteractiveShell.colors = 'NoColor' set_pandas_options() IP = IPython.InteractiveShell.instance(config=config) IP.run_cell(IMPORT_PANDAS + '\n') IP.run_cell('import numpy as np\n') try: stdout = sys.stdout for lineno, src in example_srcs: sys.stdout = cout = StringIO() src = '\n'.join(src) if src == IMPORT_PANDAS: continue IP.run_cell(src + '\n') output = cout.getvalue() if output: # Strip the prompt. # TODO(robertwb): Figure out how to suppress this. output = re.sub(r'^Out\[\d+\]:\s*', '', output) examples.append(doctest.Example(src, output, lineno=lineno)) finally: sys.stdout = stdout return doctest.DocTest(examples, dict(extraglobs or {}, np=np), name, name, None, None)
def test_group(self, group): # type: (TestGroup) -> None ns = {} # type: Dict def run_setup_cleanup(runner, testcodes, what): # type: (Any, List[TestCode], Any) -> bool examples = [] for testcode in testcodes: examples.append( doctest.Example( # type: ignore doctest_encode(testcode.code, self.env.config.source_encoding), '', # type: ignore # NOQA lineno=testcode.lineno)) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest(examples, {}, '%s (%s code)' % (group.name, what), testcodes[0].filename, 0, None) sim_doctest.globs = ns old_f = runner.failures self.type = 'exec' # the snippet may contain multiple statements runner.run(sim_doctest, out=self._warn_out, clear_globs=False) if runner.failures > old_f: return False return True # run the setup code if not run_setup_cleanup(self.setup_runner, group.setup, 'setup'): # if setup failed, don't run the group return # run the tests for code in group.tests: if len(code) == 1: # ordinary doctests (code/output interleaved) try: test = parser.get_doctest( # type: ignore doctest_encode(code[0].code, self.env.config.source_encoding), {}, # type: ignore # NOQA group.name, code[0].filename, code[0].lineno) except Exception: logger.warning(__('ignoring invalid doctest code: %r'), code[0].code, location=(code[0].filename, code[0].lineno)) continue if not test.examples: continue for example in test.examples: # apply directive's comparison options new_opt = code[0].options.copy() new_opt.update(example.options) example.options = new_opt self.type = 'single' # as for ordinary doctests else: # testcode and output separate output = code[1] and code[1].code or '' options = code[1] and code[1].options or {} # disable <BLANKLINE> processing as it is not needed options[doctest.DONT_ACCEPT_BLANKLINE] = True # find out if we're testing an exception m = parser._EXCEPTION_RE.match(output) # type: ignore if m: exc_msg = m.group('msg') else: exc_msg = None example = doctest.Example( # type: ignore doctest_encode(code[0].code, self.env.config.source_encoding), output, # type: ignore # NOQA exc_msg=exc_msg, lineno=code[0].lineno, options=options) test = doctest.DocTest( [example], {}, group.name, # type: ignore code[0].filename, code[0].lineno, None) self.type = 'exec' # multiple statements again # DocTest.__init__ copies the globs namespace, which we don't want test.globs = ns # also don't clear the globs namespace after running the doctest self.test_runner.run(test, out=self._warn_out, clear_globs=False) # run the cleanup run_setup_cleanup(self.cleanup_runner, group.cleanup, 'cleanup')
def test_group(self, group: extdoctest.TestGroup) -> None: ns: Mapping = self._make_group_globals(group) def run_setup_cleanup(runner, testcodes, what): # type: (Any, List[TestCode], Any) -> bool examples = [] for testcode in testcodes: example = doctest.Example(testcode.code, "", lineno=testcode.lineno) examples.append(example) if not examples: return True # simulate a doctest with the code sim_doctest = doctest.DocTest( examples, {}, "%s (%s code)" % (group.name, what), testcodes[0].filename, 0, None, ) sim_doctest.globs = ns old_f = runner.failures self.type = "exec" # the snippet may contain multiple statements runner.run(sim_doctest, out=self._warn_out, clear_globs=False) if runner.failures > old_f: return False return True # run the setup code if not run_setup_cleanup(self.setup_runner, group.setup, "setup"): # if setup failed, don't run the group return # run the tests for code in group.tests: py_code = code[0] if len(code) == 1: # ordinary doctests (code/output interleaved) try: test = extdoctest.parser.get_doctest( py_code.code, {}, group.name, # type: ignore py_code.filename, py_code.lineno, ) except Exception as ex: log.warning( __("ignoring invalid doctest code: %r\n due to: %s"), py_code.code, ex, location=(py_code.filename, py_code.lineno), ) continue # HACK: allow collecting vars even if code empty.. if not test.examples and not self.run_empty_code: continue for example in test.examples: # apply directive's comparison options new_opt = py_code.options.copy() new_opt.update(example.options) example.options = new_opt self.type = "single" # as for ordinary doctests else: # testcode and output separate output = code[1] and code[1].code or "" options = code[1] and code[1].options or {} # disable <BLANKLINE> processing as it is not needed options[doctest.DONT_ACCEPT_BLANKLINE] = True # find out if we're testing an exception m = extdoctest.parser._EXCEPTION_RE.match( output) # type: ignore if m: exc_msg = m.group("msg") else: exc_msg = None example = doctest.Example( py_code.code, output, exc_msg=exc_msg, lineno=py_code.lineno, options=options, ) test = doctest.DocTest([example], {}, group.name, py_code.filename, py_code.lineno, None) self.type = "exec" # multiple statements again # DocTest.__init__ copies the globs namespace, which we don't want test.globs = ns # also don't clear the globs namespace after running the doctest self.test_runner.run(test, out=self._warn_out, clear_globs=False) ## HACK: collect plottable from doctest-runner globals. self._globals_updated(py_code, ns) # run the cleanup run_setup_cleanup(self.cleanup_runner, group.cleanup, "cleanup")