def test_output_no_ansi(gallery_conf, script_vars): """Test ANSI characters are removed. See: https://en.wikipedia.org/wiki/ANSI_escape_code """ gallery_conf.update(image_scrapers=()) compiler = codeop.Compile() code = 'print("\033[94m0.25")' code_block = ("code", code, 1) output = sg.execute_code_block(compiler, code_block, None, script_vars, gallery_conf) output_test_string = "\n".join( [line[4:] for line in output.strip().split("\n")[-3:]]) assert output_test_string.split('\n')[-1] == "0.25"
def test_capture_repr(gallery_conf, capture_repr, code, expected_out): """Tests output capturing with various capture_repr settings.""" compiler = codeop.Compile() code_block = ('code', code, 1) script_vars = { "execute_script": True, "image_path_iterator": ImagePathIterator("temp.png"), "src_file": __file__, "memory_delta": [], } gallery_conf['capture_repr'] = capture_repr output = sg.execute_code_block( compiler, code_block, {}, script_vars, gallery_conf ) assert _clean_output(output) == expected_out
def test_empty_output_box(gallery_conf): """Tests that `print(__doc__)` doesn't produce an empty output box.""" compiler = codeop.Compile() code_block = ("code", "print(__doc__)", 1) script_vars = { "execute_script": True, "image_path_iterator": ImagePathIterator("temp.png"), "src_file": __file__, "memory_delta": [], } example_globals = {'__doc__': ''} output = sg.execute_code_block(compiler, code_block, example_globals, script_vars, gallery_conf) assert output.isspace()
def test_output_indentation(gallery_conf, script_vars): """Test whether indentation of code output is retained.""" gallery_conf.update(image_scrapers=()) compiler = codeop.Compile() test_string = r"\n".join([ " A B", "A 1 2", "B 3 4" ]) code = "print('" + test_string + "')" code_block = ("code", code, 1) output = sg.execute_code_block( compiler, code_block, None, script_vars, gallery_conf ) output_test_string = "\n".join( [line[4:] for line in output.strip().split("\n")[-3:]] ) assert output_test_string == test_string.replace(r"\n", "\n")
def test_output_indentation(gallery_conf): """Test whether indentation of code output is retained.""" compiler = codeop.Compile() test_string = r"\n".join([" A B", "A 1 2", "B 3 4"]) code = "print('" + test_string + "')" code_block = ("code", code, 1) script_vars = { "execute_script": True, "image_path_iterator": ImagePathIterator("temp.png"), "src_file": __file__, "memory_delta": [], } output = sg.execute_code_block(compiler, code_block, {}, script_vars, gallery_conf) output_test_string = "\n".join( [line[4:] for line in output.strip().split("\n")[-3:]]) assert output_test_string == test_string.replace(r"\n", "\n")
def _create_tutorial_section(fname, src_dir, target_dir): if not os.path.exists(target_dir): os.makedirs(target_dir) src_file = os.path.normpath(os.path.join(src_dir, fname)) # Check if the same tutorial script has been already run md5_file = os.path.join(target_dir, f"{fname}.md5") if _md5sum_is_current(src_file, md5_file): return file_conf, script_blocks = parser.split_code_and_text_blocks(src_file) # Remove *.py suffix base_image_name = os.path.splitext(fname)[0] # Locate file in tutorial target directory abs_base_image_name = os.path.join(os.getcwd(), "tutorial", "target", base_image_name) image_path_template = abs_base_image_name + "_{0:02}.png" fake_main = module_from_spec(spec_from_loader('__main__', None)) script_vars = { "execute_script": True, "image_path_iterator": scrapers.ImagePathIterator(image_path_template), "src_file": src_file, "memory_delta": [], "fake_main": fake_main } tutorial_globals = fake_main.__dict__ tutorial_globals.update({ "__doc__": "", }) gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF) gallery_conf.update({ "abort_on_example_error": True, "src_dir": os.getcwd(), "execute_script": True, "inspect_global_variables": False, "call_memory": (lambda func: (0., func())), "image_scrapers": (scrapers.matplotlib_scraper, ), }) compiler = codeop.Compile() content_rst = "" for block_label, block_content, line_no in script_blocks: if block_label == "code": # Run code and save output images code_output = genrst.execute_code_block( compiler=compiler, block=(block_label, block_content, line_no), example_globals=tutorial_globals, script_vars=script_vars, gallery_conf=gallery_conf) content_rst += genrst.codestr2rst(block_content, lineno=None) + "\n" content_rst += code_output else: content_rst += block_content + "\n\n" with open(os.path.join(target_dir, f"{base_image_name}.rst"), "w") as file: file.write(content_rst) # Write checksum of file to avoid unnecessary rerun with open(md5_file, "w") as file: file.write(genrst.get_md5sum(src_file))
def execute_script(script_blocks, script_vars, gallery_conf): """Execute and capture output from python script already in block structure Parameters ---------- script_blocks : list (label, content, line_number) List where each element is a tuple with the label ('text' or 'code'), the corresponding content string of block and the leading line number script_vars : dict Configuration and run time variables gallery_conf : dict Contains the configuration of Sphinx-Gallery Returns ------- output_blocks : list List of strings where each element is the restructured text representation of the output of each block time_elapsed : float Time elapsed during execution """ # Examples may contain if __name__ == '__main__' guards # for in example scikit-learn if the example uses multiprocessing. # Here we create a new __main__ module, and temporarily change # sys.modules when running our example fake_main = importlib.util.module_from_spec( importlib.util.spec_from_loader('__main__', None)) example_globals = fake_main.__dict__ example_globals.update({ # A lot of examples contains 'print(__doc__)' for example in # scikit-learn so that running the example prints some useful # information. Because the docstring has been separated from # the code blocks in sphinx-gallery, __doc__ is actually # __builtin__.__doc__ in the execution context and we do not # want to print it '__doc__': '', # Don't ever support __file__: Issues #166 #212 # Don't let them use input() 'input': _check_input, }) script_vars['example_globals'] = example_globals argv_orig = sys.argv[:] if script_vars['execute_script']: # We want to run the example without arguments. See # https://github.com/sphinx-gallery/sphinx-gallery/pull/252 # for more details. sys.argv[0] = script_vars['src_file'] sys.argv[1:] = gallery_conf['reset_argv'](gallery_conf, script_vars) gc.collect() memory_start, _ = gallery_conf['call_memory'](lambda: None) else: memory_start = 0. t_start = time() compiler = codeop.Compile() # include at least one entry to avoid max() ever failing script_vars['memory_delta'] = [memory_start] script_vars['fake_main'] = fake_main output_blocks = list() with _LoggingTee(script_vars.get('src_file', '')) as logging_tee: for block in script_blocks: logging_tee.set_std_and_reset_position() output_blocks.append(execute_code_block( compiler, block, example_globals, script_vars, gallery_conf)) time_elapsed = time() - t_start sys.argv = argv_orig script_vars['memory_delta'] = max(script_vars['memory_delta']) if script_vars['execute_script']: script_vars['memory_delta'] -= memory_start # Write md5 checksum if the example was meant to run (no-plot # shall not cache md5sum) and has built correctly with open(script_vars['target_file'] + '.md5', 'w') as file_checksum: file_checksum.write(get_md5sum(script_vars['target_file'], 't')) gallery_conf['passing_examples'].append(script_vars['src_file']) return output_blocks, time_elapsed
def execute_script(script_blocks, script_vars, gallery_conf): """Execute and capture output from python script already in block structure Parameters ---------- script_blocks : list (label, content, line_number) List where each element is a tuple with the label ('text' or 'code'), the corresponding content string of block and the leading line number script_vars : dict Configuration and run time variables gallery_conf : dict Contains the configuration of Sphinx-Gallery Returns ------- output_blocks : list List of strings where each element is the restructured text representation of the output of each block time_elapsed : float Time elapsed during execution """ example_globals = { # A lot of examples contains 'print(__doc__)' for example in # scikit-learn so that running the example prints some useful # information. Because the docstring has been separated from # the code blocks in sphinx-gallery, __doc__ is actually # __builtin__.__doc__ in the execution context and we do not # want to print it '__doc__': '', # Examples may contain if __name__ == '__main__' guards # for in example scikit-learn if the example uses multiprocessing '__name__': '__main__', # Don't ever support __file__: Issues #166 #212 } argv_orig = sys.argv[:] if script_vars['execute_script']: # We want to run the example without arguments. See # https://github.com/sphinx-gallery/sphinx-gallery/pull/252 # for more details. sys.argv[0] = script_vars['src_file'] sys.argv[1:] = [] t_start = time() gc.collect() _, memory_start = _memory_usage(lambda: None, gallery_conf) compiler = codeop.Compile() # include at least one entry to avoid max() ever failing script_vars['memory_delta'] = [memory_start] output_blocks = [ execute_code_block(compiler, block, example_globals, script_vars, gallery_conf) for block in script_blocks ] time_elapsed = time() - t_start script_vars['memory_delta'] = ( # actually turn it into a delta now max(script_vars['memory_delta']) - memory_start) sys.argv = argv_orig # Write md5 checksum if the example was meant to run (no-plot # shall not cache md5sum) and has built correctly if script_vars['execute_script']: with open(script_vars['target_file'] + '.md5', 'w') as file_checksum: file_checksum.write(get_md5sum(script_vars['target_file'])) gallery_conf['passing_examples'].append(script_vars['src_file']) return output_blocks, time_elapsed
def generate_file_rst(fname, target_dir, src_dir, gallery_conf): """Generate the rst file for a given example. Returns ------- intro: str The introduction of the example time_elapsed : float seconds required to run the script """ binder_conf = check_binder_conf(gallery_conf.get('binder')) src_file = os.path.normpath(os.path.join(src_dir, fname)) example_file = os.path.join(target_dir, fname) shutil.copyfile(src_file, example_file) file_conf, script_blocks = split_code_and_text_blocks(src_file) intro, title = extract_intro_and_title(fname, script_blocks[0][1]) if md5sum_is_current(example_file): return intro, 0 image_dir = os.path.join(target_dir, 'images') if not os.path.exists(image_dir): os.makedirs(image_dir) base_image_name = os.path.splitext(fname)[0] image_fname = 'sphx_glr_' + base_image_name + '_{0:03}.png' image_path_template = os.path.join(image_dir, image_fname) ref_fname = os.path.relpath(example_file, gallery_conf['src_dir']) ref_fname = ref_fname.replace(os.path.sep, '_') example_rst = """\n\n.. _sphx_glr_{0}:\n\n""".format(ref_fname) filename_pattern = gallery_conf.get('filename_pattern') execute_script = re.search(filename_pattern, src_file) and gallery_conf['plot_gallery'] example_globals = { # A lot of examples contains 'print(__doc__)' for example in # scikit-learn so that running the example prints some useful # information. Because the docstring has been separated from # the code blocks in sphinx-gallery, __doc__ is actually # __builtin__.__doc__ in the execution context and we do not # want to print it '__doc__': '', # Examples may contain if __name__ == '__main__' guards # for in example scikit-learn if the example uses multiprocessing '__name__': '__main__', # Don't ever support __file__: Issues #166 #212 } compiler = codeop.Compile() # A simple example has two blocks: one for the # example introduction/explanation and one for the code is_example_notebook_like = len(script_blocks) > 2 time_elapsed = 0 block_vars = { 'execute_script': execute_script, 'fig_count': 0, 'image_path': image_path_template, 'src_file': src_file } argv_orig = sys.argv[:] if block_vars['execute_script']: # We want to run the example without arguments. See # https://github.com/sphinx-gallery/sphinx-gallery/pull/252 # for more details. sys.argv[0] = src_file sys.argv[1:] = [] for blabel, bcontent, lineno in script_blocks: if blabel == 'code': code_output, rtime = execute_code_block(compiler, src_file, bcontent, lineno, example_globals, block_vars, gallery_conf) time_elapsed += rtime if not file_conf.get('line_numbers', gallery_conf.get('line_numbers', False)): lineno = None if is_example_notebook_like: example_rst += codestr2rst(bcontent, lineno=lineno) + '\n' example_rst += code_output else: example_rst += code_output if 'sphx-glr-script-out' in code_output: # Add some vertical space after output example_rst += "\n\n|\n\n" example_rst += codestr2rst(bcontent, lineno=lineno) + '\n' else: example_rst += bcontent + '\n\n' sys.argv = argv_orig clean_modules() # Writes md5 checksum if example has build correctly # not failed and was initially meant to run(no-plot shall not cache md5sum) if block_vars['execute_script']: with open(example_file + '.md5', 'w') as file_checksum: file_checksum.write(get_md5sum(example_file)) save_thumbnail(image_path_template, src_file, file_conf, gallery_conf) time_m, time_s = divmod(time_elapsed, 60) example_nb = jupyter_notebook(script_blocks) save_notebook(example_nb, replace_py_ipynb(example_file)) with codecs.open(os.path.join(target_dir, base_image_name + '.rst'), mode='w', encoding='utf-8') as f: if time_elapsed >= gallery_conf["min_reported_time"]: example_rst += ("**Total running time of the script:**" " ({0: .0f} minutes {1: .3f} seconds)\n\n".format( time_m, time_s)) # Generate a binder URL if specified binder_badge_rst = '' if len(binder_conf) > 0: binder_badge_rst += gen_binder_rst(fname, binder_conf) example_rst += CODE_DOWNLOAD.format(fname, replace_py_ipynb(fname), binder_badge_rst) example_rst += SPHX_GLR_SIG f.write(example_rst) if block_vars['execute_script']: logger.debug("%s ran in : %.2g seconds", src_file, time_elapsed) return intro, time_elapsed
def _create_tutorial_section(fname, src_dir, target_dir): if not os.path.exists(target_dir): os.makedirs(target_dir) src_file = os.path.normpath(os.path.join(src_dir, fname)) # Check if the same tutorial script has been already run md5_file = os.path.join(target_dir, f"{fname}.md5") if _md5sum_is_current(src_file, md5_file): return file_conf, script_blocks = parser.split_code_and_text_blocks(src_file) # Remove *.py suffix base_image_name = os.path.splitext(fname)[0] image_path_template = os.path.join(target_dir, base_image_name+"_{0:02}.png") script_vars = { "execute_script": True, "image_path_iterator": scrapers.ImagePathIterator(image_path_template), "src_file": src_file, "memory_delta": [], } tutorial_globals = { "__doc__": "", "__name__": "__main__" } compiler = codeop.Compile() content_rst = "" for block_label, block_content, line_no in script_blocks: if block_label == "code": # Run code and save output images code_output = genrst.execute_code_block( compiler=compiler, block=(block_label, block_content, line_no), example_globals=tutorial_globals, script_vars=script_vars, gallery_conf = { "abort_on_example_error": True, "src_dir": ".", "execute_script": True, "show_memory": False, "capture_repr": (), "image_scrapers": (scrapers.matplotlib_scraper,), } ) content_rst += genrst.codestr2rst( block_content, lineno=None ) + "\n" content_rst += code_output else: content_rst += block_content + "\n\n" with open(os.path.join(target_dir, f"{base_image_name}.rst"), "w") as file: file.write(content_rst) # Write checksum of file to avoid unnecessary rerun with open(md5_file, "w") as file: file.write(genrst.get_md5sum(src_file))
def __init__(self): import codeop self._pybuf = "" self._compile = codeop.Compile()