def test_output_no_ansi(gallery_conf, script_vars):
"""Test ANSI characters are removed.
See: https://en.wikipedia.org/wiki/ANSI_escape_code
"""
gallery_conf.update(image_scrapers=())
compiler = codeop.Compile()
code = 'print("\033[94m0.25")'
code_block = ("code", code, 1)
output = sg.execute_code_block(compiler, code_block, None, script_vars,
gallery_conf)
output_test_string = "\n".join(
[line[4:] for line in output.strip().split("\n")[-3:]])
assert output_test_string.split('\n')[-1] == "0.25"
def test_capture_repr(gallery_conf, capture_repr, code, expected_out):
"""Tests output capturing with various capture_repr settings."""
compiler = codeop.Compile()
code_block = ('code', code, 1)
script_vars = {
"execute_script": True,
"image_path_iterator": ImagePathIterator("temp.png"),
"src_file": __file__,
"memory_delta": [],
}
gallery_conf['capture_repr'] = capture_repr
output = sg.execute_code_block(
compiler, code_block, {}, script_vars, gallery_conf
)
assert _clean_output(output) == expected_out
def test_empty_output_box(gallery_conf):
"""Tests that `print(__doc__)` doesn't produce an empty output box."""
compiler = codeop.Compile()
code_block = ("code", "print(__doc__)", 1)
script_vars = {
"execute_script": True,
"image_path_iterator": ImagePathIterator("temp.png"),
"src_file": __file__,
"memory_delta": [],
}
example_globals = {'__doc__': ''}
output = sg.execute_code_block(compiler, code_block, example_globals,
script_vars, gallery_conf)
assert output.isspace()
def test_output_indentation(gallery_conf, script_vars):
"""Test whether indentation of code output is retained."""
gallery_conf.update(image_scrapers=())
compiler = codeop.Compile()
test_string = r"\n".join([
" A B",
"A 1 2",
"B 3 4"
])
code = "print('" + test_string + "')"
code_block = ("code", code, 1)
output = sg.execute_code_block(
compiler, code_block, None, script_vars, gallery_conf
)
output_test_string = "\n".join(
[line[4:] for line in output.strip().split("\n")[-3:]]
)
assert output_test_string == test_string.replace(r"\n", "\n")
def test_output_indentation(gallery_conf):
"""Test whether indentation of code output is retained."""
compiler = codeop.Compile()
test_string = r"\n".join([" A B", "A 1 2", "B 3 4"])
code = "print('" + test_string + "')"
code_block = ("code", code, 1)
script_vars = {
"execute_script": True,
"image_path_iterator": ImagePathIterator("temp.png"),
"src_file": __file__,
"memory_delta": [],
}
output = sg.execute_code_block(compiler, code_block, {}, script_vars,
gallery_conf)
output_test_string = "\n".join(
[line[4:] for line in output.strip().split("\n")[-3:]])
assert output_test_string == test_string.replace(r"\n", "\n")
def _create_tutorial_section(fname, src_dir, target_dir):
if not os.path.exists(target_dir):
os.makedirs(target_dir)
src_file = os.path.normpath(os.path.join(src_dir, fname))
# Check if the same tutorial script has been already run
md5_file = os.path.join(target_dir, f"{fname}.md5")
if _md5sum_is_current(src_file, md5_file):
return
file_conf, script_blocks = parser.split_code_and_text_blocks(src_file)
# Remove *.py suffix
base_image_name = os.path.splitext(fname)[0]
# Locate file in tutorial target directory
abs_base_image_name = os.path.join(os.getcwd(), "tutorial", "target",
base_image_name)
image_path_template = abs_base_image_name + "_{0:02}.png"
fake_main = module_from_spec(spec_from_loader('__main__', None))
script_vars = {
"execute_script": True,
"image_path_iterator": scrapers.ImagePathIterator(image_path_template),
"src_file": src_file,
"memory_delta": [],
"fake_main": fake_main
}
tutorial_globals = fake_main.__dict__
tutorial_globals.update({
"__doc__": "",
})
gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF)
gallery_conf.update({
"abort_on_example_error": True,
"src_dir": os.getcwd(),
"execute_script": True,
"inspect_global_variables": False,
"call_memory": (lambda func: (0., func())),
"image_scrapers": (scrapers.matplotlib_scraper, ),
})
compiler = codeop.Compile()
content_rst = ""
for block_label, block_content, line_no in script_blocks:
if block_label == "code":
# Run code and save output images
code_output = genrst.execute_code_block(
compiler=compiler,
block=(block_label, block_content, line_no),
example_globals=tutorial_globals,
script_vars=script_vars,
gallery_conf=gallery_conf)
content_rst += genrst.codestr2rst(block_content,
lineno=None) + "\n"
content_rst += code_output
else:
content_rst += block_content + "\n\n"
with open(os.path.join(target_dir, f"{base_image_name}.rst"), "w") as file:
file.write(content_rst)
# Write checksum of file to avoid unnecessary rerun
with open(md5_file, "w") as file:
file.write(genrst.get_md5sum(src_file))
def execute_script(script_blocks, script_vars, gallery_conf):
"""Execute and capture output from python script already in block structure
Parameters
----------
script_blocks : list
(label, content, line_number)
List where each element is a tuple with the label ('text' or 'code'),
the corresponding content string of block and the leading line number
script_vars : dict
Configuration and run time variables
gallery_conf : dict
Contains the configuration of Sphinx-Gallery
Returns
-------
output_blocks : list
List of strings where each element is the restructured text
representation of the output of each block
time_elapsed : float
Time elapsed during execution
"""
# Examples may contain if __name__ == '__main__' guards
# for in example scikit-learn if the example uses multiprocessing.
# Here we create a new __main__ module, and temporarily change
# sys.modules when running our example
fake_main = importlib.util.module_from_spec(
importlib.util.spec_from_loader('__main__', None))
example_globals = fake_main.__dict__
example_globals.update({
# A lot of examples contains 'print(__doc__)' for example in
# scikit-learn so that running the example prints some useful
# information. Because the docstring has been separated from
# the code blocks in sphinx-gallery, __doc__ is actually
# __builtin__.__doc__ in the execution context and we do not
# want to print it
'__doc__': '',
# Don't ever support __file__: Issues #166 #212
# Don't let them use input()
'input': _check_input,
})
script_vars['example_globals'] = example_globals
argv_orig = sys.argv[:]
if script_vars['execute_script']:
# We want to run the example without arguments. See
# https://github.com/sphinx-gallery/sphinx-gallery/pull/252
# for more details.
sys.argv[0] = script_vars['src_file']
sys.argv[1:] = gallery_conf['reset_argv'](gallery_conf, script_vars)
gc.collect()
memory_start, _ = gallery_conf['call_memory'](lambda: None)
else:
memory_start = 0.
t_start = time()
compiler = codeop.Compile()
# include at least one entry to avoid max() ever failing
script_vars['memory_delta'] = [memory_start]
script_vars['fake_main'] = fake_main
output_blocks = list()
with _LoggingTee(script_vars.get('src_file', '')) as logging_tee:
for block in script_blocks:
logging_tee.set_std_and_reset_position()
output_blocks.append(execute_code_block(
compiler, block, example_globals, script_vars, gallery_conf))
time_elapsed = time() - t_start
sys.argv = argv_orig
script_vars['memory_delta'] = max(script_vars['memory_delta'])
if script_vars['execute_script']:
script_vars['memory_delta'] -= memory_start
# Write md5 checksum if the example was meant to run (no-plot
# shall not cache md5sum) and has built correctly
with open(script_vars['target_file'] + '.md5', 'w') as file_checksum:
file_checksum.write(get_md5sum(script_vars['target_file'], 't'))
gallery_conf['passing_examples'].append(script_vars['src_file'])
return output_blocks, time_elapsed
def execute_script(script_blocks, script_vars, gallery_conf):
"""Execute and capture output from python script already in block structure
Parameters
----------
script_blocks : list
(label, content, line_number)
List where each element is a tuple with the label ('text' or 'code'),
the corresponding content string of block and the leading line number
script_vars : dict
Configuration and run time variables
gallery_conf : dict
Contains the configuration of Sphinx-Gallery
Returns
-------
output_blocks : list
List of strings where each element is the restructured text
representation of the output of each block
time_elapsed : float
Time elapsed during execution
"""
example_globals = {
# A lot of examples contains 'print(__doc__)' for example in
# scikit-learn so that running the example prints some useful
# information. Because the docstring has been separated from
# the code blocks in sphinx-gallery, __doc__ is actually
# __builtin__.__doc__ in the execution context and we do not
# want to print it
'__doc__': '',
# Examples may contain if __name__ == '__main__' guards
# for in example scikit-learn if the example uses multiprocessing
'__name__': '__main__',
# Don't ever support __file__: Issues #166 #212
}
argv_orig = sys.argv[:]
if script_vars['execute_script']:
# We want to run the example without arguments. See
# https://github.com/sphinx-gallery/sphinx-gallery/pull/252
# for more details.
sys.argv[0] = script_vars['src_file']
sys.argv[1:] = []
t_start = time()
gc.collect()
_, memory_start = _memory_usage(lambda: None, gallery_conf)
compiler = codeop.Compile()
# include at least one entry to avoid max() ever failing
script_vars['memory_delta'] = [memory_start]
output_blocks = [
execute_code_block(compiler, block, example_globals, script_vars,
gallery_conf) for block in script_blocks
]
time_elapsed = time() - t_start
script_vars['memory_delta'] = ( # actually turn it into a delta now
max(script_vars['memory_delta']) - memory_start)
sys.argv = argv_orig
# Write md5 checksum if the example was meant to run (no-plot
# shall not cache md5sum) and has built correctly
if script_vars['execute_script']:
with open(script_vars['target_file'] + '.md5', 'w') as file_checksum:
file_checksum.write(get_md5sum(script_vars['target_file']))
gallery_conf['passing_examples'].append(script_vars['src_file'])
return output_blocks, time_elapsed
def generate_file_rst(fname, target_dir, src_dir, gallery_conf):
"""Generate the rst file for a given example.
Returns
-------
intro: str
The introduction of the example
time_elapsed : float
seconds required to run the script
"""
binder_conf = check_binder_conf(gallery_conf.get('binder'))
src_file = os.path.normpath(os.path.join(src_dir, fname))
example_file = os.path.join(target_dir, fname)
shutil.copyfile(src_file, example_file)
file_conf, script_blocks = split_code_and_text_blocks(src_file)
intro, title = extract_intro_and_title(fname, script_blocks[0][1])
if md5sum_is_current(example_file):
return intro, 0
image_dir = os.path.join(target_dir, 'images')
if not os.path.exists(image_dir):
os.makedirs(image_dir)
base_image_name = os.path.splitext(fname)[0]
image_fname = 'sphx_glr_' + base_image_name + '_{0:03}.png'
image_path_template = os.path.join(image_dir, image_fname)
ref_fname = os.path.relpath(example_file, gallery_conf['src_dir'])
ref_fname = ref_fname.replace(os.path.sep, '_')
example_rst = """\n\n.. _sphx_glr_{0}:\n\n""".format(ref_fname)
filename_pattern = gallery_conf.get('filename_pattern')
execute_script = re.search(filename_pattern,
src_file) and gallery_conf['plot_gallery']
example_globals = {
# A lot of examples contains 'print(__doc__)' for example in
# scikit-learn so that running the example prints some useful
# information. Because the docstring has been separated from
# the code blocks in sphinx-gallery, __doc__ is actually
# __builtin__.__doc__ in the execution context and we do not
# want to print it
'__doc__': '',
# Examples may contain if __name__ == '__main__' guards
# for in example scikit-learn if the example uses multiprocessing
'__name__': '__main__',
# Don't ever support __file__: Issues #166 #212
}
compiler = codeop.Compile()
# A simple example has two blocks: one for the
# example introduction/explanation and one for the code
is_example_notebook_like = len(script_blocks) > 2
time_elapsed = 0
block_vars = {
'execute_script': execute_script,
'fig_count': 0,
'image_path': image_path_template,
'src_file': src_file
}
argv_orig = sys.argv[:]
if block_vars['execute_script']:
# We want to run the example without arguments. See
# https://github.com/sphinx-gallery/sphinx-gallery/pull/252
# for more details.
sys.argv[0] = src_file
sys.argv[1:] = []
for blabel, bcontent, lineno in script_blocks:
if blabel == 'code':
code_output, rtime = execute_code_block(compiler, src_file,
bcontent, lineno,
example_globals,
block_vars, gallery_conf)
time_elapsed += rtime
if not file_conf.get('line_numbers',
gallery_conf.get('line_numbers', False)):
lineno = None
if is_example_notebook_like:
example_rst += codestr2rst(bcontent, lineno=lineno) + '\n'
example_rst += code_output
else:
example_rst += code_output
if 'sphx-glr-script-out' in code_output:
# Add some vertical space after output
example_rst += "\n\n|\n\n"
example_rst += codestr2rst(bcontent, lineno=lineno) + '\n'
else:
example_rst += bcontent + '\n\n'
sys.argv = argv_orig
clean_modules()
# Writes md5 checksum if example has build correctly
# not failed and was initially meant to run(no-plot shall not cache md5sum)
if block_vars['execute_script']:
with open(example_file + '.md5', 'w') as file_checksum:
file_checksum.write(get_md5sum(example_file))
save_thumbnail(image_path_template, src_file, file_conf, gallery_conf)
time_m, time_s = divmod(time_elapsed, 60)
example_nb = jupyter_notebook(script_blocks)
save_notebook(example_nb, replace_py_ipynb(example_file))
with codecs.open(os.path.join(target_dir, base_image_name + '.rst'),
mode='w',
encoding='utf-8') as f:
if time_elapsed >= gallery_conf["min_reported_time"]:
example_rst += ("**Total running time of the script:**"
" ({0: .0f} minutes {1: .3f} seconds)\n\n".format(
time_m, time_s))
# Generate a binder URL if specified
binder_badge_rst = ''
if len(binder_conf) > 0:
binder_badge_rst += gen_binder_rst(fname, binder_conf)
example_rst += CODE_DOWNLOAD.format(fname, replace_py_ipynb(fname),
binder_badge_rst)
example_rst += SPHX_GLR_SIG
f.write(example_rst)
if block_vars['execute_script']:
logger.debug("%s ran in : %.2g seconds", src_file, time_elapsed)
return intro, time_elapsed
def _create_tutorial_section(fname, src_dir, target_dir):
if not os.path.exists(target_dir):
os.makedirs(target_dir)
src_file = os.path.normpath(os.path.join(src_dir, fname))
# Check if the same tutorial script has been already run
md5_file = os.path.join(target_dir, f"{fname}.md5")
if _md5sum_is_current(src_file, md5_file):
return
file_conf, script_blocks = parser.split_code_and_text_blocks(src_file)
# Remove *.py suffix
base_image_name = os.path.splitext(fname)[0]
image_path_template = os.path.join(target_dir,
base_image_name+"_{0:02}.png")
script_vars = {
"execute_script": True,
"image_path_iterator": scrapers.ImagePathIterator(image_path_template),
"src_file": src_file,
"memory_delta": [],
}
tutorial_globals = {
"__doc__": "",
"__name__": "__main__"
}
compiler = codeop.Compile()
content_rst = ""
for block_label, block_content, line_no in script_blocks:
if block_label == "code":
# Run code and save output images
code_output = genrst.execute_code_block(
compiler=compiler,
block=(block_label, block_content, line_no),
example_globals=tutorial_globals,
script_vars=script_vars,
gallery_conf = {
"abort_on_example_error": True,
"src_dir": ".",
"execute_script": True,
"show_memory": False,
"capture_repr": (),
"image_scrapers": (scrapers.matplotlib_scraper,),
}
)
content_rst += genrst.codestr2rst(
block_content, lineno=None
) + "\n"
content_rst += code_output
else:
content_rst += block_content + "\n\n"
with open(os.path.join(target_dir, f"{base_image_name}.rst"), "w") as file:
file.write(content_rst)
# Write checksum of file to avoid unnecessary rerun
with open(md5_file, "w") as file:
file.write(genrst.get_md5sum(src_file))
def __init__(self):
import codeop
self._pybuf = ""
self._compile = codeop.Compile()
