def test_auto_in_formats(nb_file):
nb = read(nb_file)
nb.metadata["jupytext"] = {"formats": "ipynb,auto:percent"}
fmt = auto_ext_from_metadata(nb.metadata)[1:] + ":percent"
expected_formats = "ipynb," + fmt
text = writes(nb, "ipynb")
assert "auto" not in text
nb2 = reads(text, "ipynb")
assert nb2.metadata["jupytext"]["formats"] == expected_formats
text = writes(nb, "auto:percent")
assert "auto" not in text
nb2 = reads(text, fmt)
assert nb2.metadata["jupytext"]["formats"] == expected_formats
del nb.metadata["language_info"]
del nb.metadata["kernelspec"]
with pytest.raises(JupytextFormatError):
writes(nb, "ipynb")
with pytest.raises(JupytextFormatError):
writes(nb, "auto:percent")
def test_pipe_nbconvert_execute_sync(tmpdir):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_py = str(tmpdir.join("notebook.py"))
with open(tmp_py, "w") as fp:
fp.write("""1 + 2
""")
jupytext(args=[
tmp_py,
"--set-formats",
"py,ipynb",
"--sync",
"--pipe-fmt",
"ipynb",
"--pipe",
"jupyter nbconvert --stdin --stdout --to notebook --execute",
])
nb = read(tmp_ipynb)
assert len(nb.cells) == 1
assert nb.cells[0].outputs[0]["data"] == {"text/plain": "3"}
def test_cell_markers_in_contents_manager_does_not_impact_light_format(tmpdir):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_py = str(tmpdir.join("notebook.py"))
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.cell_markers = "'''"
nb = new_notebook(
cells=[new_code_cell("1 + 1"),
new_markdown_cell("a\nlong\ncell")],
metadata={
"jupytext": {
"formats": "ipynb,py",
"notebook_metadata_filter": "-all"
}
},
)
with pytest.warns(UserWarning, match="Ignored cell markers"):
cm.save(model=notebook_model(nb), path="notebook.ipynb")
assert os.path.isfile(tmp_ipynb)
assert os.path.isfile(tmp_py)
with open(tmp_py) as fp:
text = fp.read()
compare(
text,
"""1 + 1
# a
# long
# cell
""",
)
nb2 = jupytext.read(tmp_py)
compare_notebooks(nb, nb2)
def test_auto_in_formats(nb_file):
nb = read(nb_file)
nb.metadata['jupytext'] = {'formats': 'ipynb,auto:percent'}
fmt = auto_ext_from_metadata(nb.metadata)[1:] + ':percent'
expected_formats = 'ipynb,' + fmt
text = writes(nb, 'ipynb')
assert 'auto' not in text
nb2 = reads(text, 'ipynb')
assert nb2.metadata['jupytext']['formats'] == expected_formats
text = writes(nb, 'auto:percent')
assert 'auto' not in text
nb2 = reads(text, fmt)
assert nb2.metadata['jupytext']['formats'] == expected_formats
del nb.metadata['language_info']
del nb.metadata['kernelspec']
with pytest.raises(JupytextFormatError):
writes(nb, 'ipynb')
with pytest.raises(JupytextFormatError):
writes(nb, 'auto:percent')
def test_load_save_rename_non_ascii_path(nb_file, tmpdir):
tmp_ipynb = u'notebôk.ipynb'
tmp_nbpy = u'notebôk.nb.py'
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'ipynb,.nb.py'
tmpdir = u'' + str(tmpdir)
cm.root_dir = tmpdir
# open ipynb, save nb.py, reopen
nb = jupytext.read(nb_file)
cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy)
nbpy = cm.get(tmp_nbpy)
compare_notebooks(nb, nbpy['content'])
# open ipynb
nbipynb = cm.get(tmp_ipynb)
compare_notebooks(nb, nbipynb['content'])
# save ipynb
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
# rename notebôk.nb.py to nêw.nb.py
cm.rename(tmp_nbpy, u'nêw.nb.py')
assert not os.path.isfile(os.path.join(tmpdir, tmp_ipynb))
assert not os.path.isfile(os.path.join(tmpdir, tmp_nbpy))
assert os.path.isfile(os.path.join(tmpdir, u'nêw.ipynb'))
assert os.path.isfile(os.path.join(tmpdir, u'nêw.nb.py'))
# rename nêw.ipynb to notebôk.ipynb
cm.rename(u'nêw.ipynb', tmp_ipynb)
assert os.path.isfile(os.path.join(tmpdir, tmp_ipynb))
assert os.path.isfile(os.path.join(tmpdir, tmp_nbpy))
assert not os.path.isfile(os.path.join(tmpdir, u'nêw.ipynb'))
assert not os.path.isfile(os.path.join(tmpdir, u'nêw.nb.py'))
def test_default_cell_markers_in_contents_manager(tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_py = str(tmpdir.join('notebook.py'))
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.default_cell_markers = "'''"
nb = new_notebook(
cells=[new_code_cell('1 + 1'),
new_markdown_cell('a\nlong\ncell')],
metadata={
'jupytext': {
'formats': 'ipynb,py:percent',
'notebook_metadata_filter': '-all'
}
})
cm.save(model=dict(type='notebook', content=nb), path='notebook.ipynb')
assert os.path.isfile(tmp_ipynb)
assert os.path.isfile(tmp_py)
with open(tmp_py) as fp:
text = fp.read()
compare(text, """# %%
1 + 1
# %% [markdown]
'''
a
long
cell
'''
""")
nb2 = jupytext.read(tmp_py)
compare_notebooks(nb, nb2)
def test_save_to_light_percent_sphinx_format(nb_file, tmpdir):
tmp_ipynb = 'notebook.ipynb'
tmp_lgt_py = 'notebook.lgt.py'
tmp_pct_py = 'notebook.pct.py'
tmp_spx_py = 'notebook.spx.py'
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
nb = jupytext.read(nb_file)
nb['metadata']['jupytext'] = {
'formats': 'ipynb,.pct.py:percent,.lgt.py:light,.spx.py:sphinx'
}
# save to ipynb and three python flavors
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
# read files
with open(str(tmpdir.join(tmp_pct_py))) as stream:
assert read_format_from_metadata(stream.read(), '.py') == 'percent'
with open(str(tmpdir.join(tmp_lgt_py))) as stream:
assert read_format_from_metadata(stream.read(), '.py') == 'light'
with open(str(tmpdir.join(tmp_spx_py))) as stream:
assert read_format_from_metadata(stream.read(), '.py') == 'sphinx'
model = cm.get(path=tmp_pct_py)
compare_notebooks(nb, model['content'])
model = cm.get(path=tmp_lgt_py)
compare_notebooks(nb, model['content'])
model = cm.get(path=tmp_spx_py)
# (notebooks not equal as we insert %matplotlib inline in sphinx)
model = cm.get(path=tmp_ipynb)
compare_notebooks(nb, model['content'])
def exe_check_nb_fname(nb_fname, wd=None):
""" Execute notebook, return error message or 'ok'
Parameters
----------
nb_fname : str
Filename of notebook to execute.
wd : None or str
Directory in which to execute notebook. None means use directory
containing `nb_fname`
Returns
-------
err_msg : str
Error message or 'ok' for no error.
"""
wd = op.dirname(nb_fname) if wd is None else wd
nb = jupytext.read(nb_fname)
try:
executenb(nb, cwd=wd, kernel_name=nb2kernel_name(nb))
except CellExecutionError as e:
return str(e)
return 'ok'
def test_outdated_text_notebook(nb_file, tmpdir):
# 1. write py ipynb
tmp_ipynb = u'notebook.ipynb'
tmp_nbpy = u'notebook.py'
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'py,ipynb'
cm.outdated_text_notebook_margin = 0
cm.root_dir = str(tmpdir)
# open ipynb, save py, reopen
nb = jupytext.read(nb_file)
cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy)
model_py = cm.get(tmp_nbpy, load_alternative_format=False)
model_ipynb = cm.get(tmp_ipynb, load_alternative_format=False)
# 2. check that time of ipynb <= py
assert model_ipynb['last_modified'] <= model_py['last_modified']
# 3. wait some time
time.sleep(0.5)
# 4. touch ipynb
with open(str(tmpdir.join(tmp_ipynb)), 'a'):
os.utime(str(tmpdir.join(tmp_ipynb)), None)
# 5. test error
with pytest.raises(HTTPError):
cm.get(tmp_nbpy)
# 6. test OK with
cm.outdated_text_notebook_margin = 1.0
cm.get(tmp_nbpy)
# 7. test OK with
cm.outdated_text_notebook_margin = float("inf")
cm.get(tmp_nbpy)
def test_pair_plain_script(py_file, tmpdir):
tmp_py = 'notebook.py'
tmp_ipynb = 'notebook.ipynb'
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
# open py file, pair, save with cm
nb = jupytext.read(py_file)
nb.metadata['jupytext']['formats'] = 'ipynb,py:hydrogen'
cm.save(model=dict(type='notebook', content=nb), path=tmp_py)
assert os.path.isfile(str(tmpdir.join(tmp_py)))
assert os.path.isfile(str(tmpdir.join(tmp_ipynb)))
# Make sure we've not changed the script
with open(py_file) as fp:
script = fp.read()
with open(str(tmpdir.join(tmp_py))) as fp:
script2 = fp.read()
compare(script, script2)
# reopen py file with the cm
nb2 = cm.get(tmp_py)['content']
compare_notebooks(nb, nb2)
assert nb2.metadata['jupytext']['formats'] == 'ipynb,py:hydrogen'
# remove the pairing and save
del nb.metadata['jupytext']['formats']
cm.save(model=dict(type='notebook', content=nb), path=tmp_py)
# reopen py file with the cm
nb2 = cm.get(tmp_py)['content']
compare_notebooks(nb, nb2)
assert 'formats' not in nb2.metadata['jupytext']
def render_rmd(input_file: str, output_file: str, params: dict = None):
"""
Wrapper function to render an Rmarkdown document with
the R `rmarkdown` package and convert it to HTML using pandoc
and a custom template.
Parameters
----------
input_file
path to input (Rmd) file
output_file
path to output (html) file
params
Dictionary that will be passed to `params` arg of `rmarkdown::render`.
See https://bookdown.org/yihui/rmarkdown/parameterized-reports.html for more details.
"""
# Directory the notebook is located in. Will be used as additional resource path for pandoc.
nb_dir = os.path.abspath(os.path.dirname(input_file))
with TemporaryDirectory() as tmp_dir:
with TemporaryDirectory() as tmp_dir_nb_converted:
# Create this file manually with given name
# so that pandoc gracefully falls back to the
# file name if no title is specified within the file. (#17)
tmp_nb_converted = os.path.join(
tmp_dir_nb_converted,
os.path.splitext(os.path.basename(input_file))[0] + ".Rmd",
)
if not input_file.endswith(".Rmd"):
nb = jtx.read(input_file)
jtx.write(nb, tmp_nb_converted)
else:
copyfile(input_file, tmp_nb_converted)
md_file = _run_rmarkdown(tmp_nb_converted, tmp_dir, params)
run_pandoc(md_file, output_file, res_path=[nb_dir, tmp_dir])
def test_load_save_rename_nbpy_default_config(nb_file, tmpdir):
tmp_ipynb = 'notebook.ipynb'
tmp_nbpy = 'notebook.nb.py'
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'ipynb,.nb.py'
cm.root_dir = str(tmpdir)
# open ipynb, save nb.py, reopen
nb = jupytext.read(nb_file)
cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy)
nbpy = cm.get(tmp_nbpy)
compare_notebooks(nbpy['content'], nb)
# open ipynb
nbipynb = cm.get(tmp_ipynb)
compare_notebooks(nbipynb['content'], nb)
# save ipynb
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
# rename notebook.nb.py to new.nb.py
cm.rename(tmp_nbpy, 'new.nb.py')
assert not os.path.isfile(str(tmpdir.join(tmp_ipynb)))
assert not os.path.isfile(str(tmpdir.join(tmp_nbpy)))
assert os.path.isfile(str(tmpdir.join('new.ipynb')))
assert os.path.isfile(str(tmpdir.join('new.nb.py')))
# rename new.ipynb to notebook.ipynb
cm.rename('new.ipynb', tmp_ipynb)
assert os.path.isfile(str(tmpdir.join(tmp_ipynb)))
assert os.path.isfile(str(tmpdir.join(tmp_nbpy)))
assert not os.path.isfile(str(tmpdir.join('new.ipynb')))
assert not os.path.isfile(str(tmpdir.join('new.nb.py')))
def test_sync_pipe_config(tmpdir):
"""Sync a notebook to a script paired in a tree, and reformat the markdown cells using pandoc"""
tmpdir.join("jupytext.toml").write(
"""# By default, the notebooks in this repository are in the notebooks subfolder
# and they are paired to scripts in the script subfolder.
default_jupytext_formats = "notebooks///ipynb,scripts///py:percent"
"""
)
nb_file = tmpdir.mkdir("notebooks").join("wrap_markdown.ipynb")
long_text = "This is a " + ("very " * 24) + "long sentence."
assert len(long_text) > 100
nb = new_notebook(cells=[new_markdown_cell(long_text)])
write(nb, str(nb_file))
jupytext(
[
"--sync",
"--pipe-fmt",
"ipynb",
"--pipe",
"pandoc --from ipynb --to ipynb --atx-headers",
str(nb_file),
]
)
py_text = tmpdir.join("scripts").join("wrap_markdown.py").read()
assert "This is a very very" in py_text
for line in py_text.splitlines():
assert len(line) <= 79
nb = read(nb_file, as_version=4)
text = nb.cells[0].source
assert len(text.splitlines()) == 3
assert text != long_text
def test_execute_in_subfolder(tmpdir):
tmpdir.mkdir('subfolder')
tmp_csv = str(tmpdir.join('subfolder', 'inputs.csv'))
tmp_py = str(tmpdir.join('subfolder', 'notebook.py'))
tmp_ipynb = str(tmpdir.join('subfolder', 'notebook.ipynb'))
with open(tmp_csv, 'w') as fp:
fp.write("1\n2\n")
with open(tmp_py, 'w') as fp:
fp.write("""import ast
with open('inputs.csv') as fp:
text = fp.read()
sum(ast.literal_eval(line) for line in text.splitlines())
""")
jupytext(args=[tmp_py, '--to', 'ipynb', '--execute'])
nb = read(tmp_ipynb)
assert len(nb.cells) == 3
assert nb.cells[2].outputs[0]['data'] == {'text/plain': '3'}
def test_execute_in_subfolder(tmpdir, caplog, capsys):
tmpdir.mkdir("subfolder")
tmp_csv = str(tmpdir.join("subfolder", "inputs.csv"))
tmp_py = str(tmpdir.join("subfolder", "notebook.py"))
tmp_ipynb = str(tmpdir.join("subfolder", "notebook.ipynb"))
with open(tmp_csv, "w") as fp:
fp.write("1\n2\n")
with open(tmp_py, "w") as fp:
fp.write("""import ast
with open('inputs.csv') as fp:
text = fp.read()
sum(ast.literal_eval(line) for line in text.splitlines())
""")
jupytext(args=[tmp_py, "--to", "ipynb", "--execute"])
nb = read(tmp_ipynb)
assert len(nb.cells) == 3
assert nb.cells[2].outputs[0]["data"] == {"text/plain": "3"}
def test_jupytext(tmpdir):
path_jupytext_dir = op.join(this_folder, 'site', 'content', 'tests', 'jupytext')
jupytext_files = {
'.md': op.join(path_jupytext_dir, 'jupytext_md.md'),
'.ipynb': op.join(path_jupytext_dir, 'jupytext_ipynb.ipynb'),
'.py': op.join(path_jupytext_dir, 'jupytext_py.py'),
'.Rmd': op.join(path_jupytext_dir, 'jupytext_Rmd.Rmd')
}
jupytext_html = {}
for ext, ifile in jupytext_files.items():
ntbk = jpt.read(ifile)
html, resources = page_html(ntbk, execute_dir=path_jupytext_dir)
jupytext_html[ext] = html
assert "tag_hide_input" in html
assert "This message should display: 4" in html
# Make sure all of the jupytext files in different formats are the same
base_ext = ".ipynb"
for compare_ext in [".md", ".py", ".Rmd"]:
assert jupytext_html[base_ext] == jupytext_html[compare_ext]
def test_rst2md(tmpdir):
tmp_py = str(tmpdir.join('notebook.py'))
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
# Write notebook in sphinx format
nb = new_notebook(cells=[
new_markdown_cell('A short sphinx notebook'),
new_markdown_cell(':math:`1+1`')
])
write(nb, tmp_py, fmt='py:sphinx')
jupytext([
tmp_py, '--from', 'py:sphinx', '--to', 'ipynb', '--opt', 'rst2md=True',
'--opt', 'cell_metadata_filter=-all'
])
assert os.path.isfile(tmp_ipynb)
nb = read(tmp_ipynb)
assert nb.metadata['jupytext']['cell_metadata_filter'] == '-all'
assert nb.metadata['jupytext']['rst2md'] is False
# Was rst to md conversion effective?
assert nb.cells[2].source == '$1+1$'
def test_page_standalone(tmpdir):
path_ipynb = op.join(this_folder, 'site', 'content', 'tests', 'notebooks.ipynb')
path_out = op.join(tmpdir.dirpath(), '.')
ntbk = jpt.read(path_ipynb)
html, resources = page_html(ntbk, execute_dir=op.dirname(path_ipynb))
custom_css = """
h1 {
font-size: REALLYBIG;
}
"""
custom_js = """
console.log("OMG")
"""
path_html = write_page(html, path_out, resources, standalone=True,
custom_css=custom_css, custom_js=custom_js)
with open(path_html, 'r') as ff:
html = ff.read()
assert "<!DOCTYPE html>" in html
assert custom_css in html
assert custom_js in html
def test_load_save_rename_notebook_with_dot(nb_file, tmpdir):
tmp_ipynb = '1.notebook.ipynb'
tmp_nbpy = '1.notebook.py'
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'ipynb,py'
cm.root_dir = str(tmpdir)
# open ipynb, save nb.py, reopen
nb = jupytext.read(nb_file)
cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy)
nbpy = cm.get(tmp_nbpy)
compare_notebooks(nb, nbpy['content'])
# save ipynb
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
# rename py
cm.rename(tmp_nbpy, '2.new_notebook.py')
assert not os.path.isfile(str(tmpdir.join(tmp_ipynb)))
assert not os.path.isfile(str(tmpdir.join(tmp_nbpy)))
assert os.path.isfile(str(tmpdir.join('2.new_notebook.ipynb')))
assert os.path.isfile(str(tmpdir.join('2.new_notebook.py')))
def test_save_in_auto_extension_local(nb_file, tmpdir):
# load notebook
nb = jupytext.read(nb_file)
nb.metadata.setdefault('jupytext', {})['formats'] = 'ipynb,auto:percent'
auto_ext = auto_ext_from_metadata(nb.metadata)
tmp_ipynb = 'notebook.ipynb'
tmp_script = 'notebook' + auto_ext
# create contents manager with default load format as percent
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
# save notebook
cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
# check that text representation exists, and is in percent format
with open(str(tmpdir.join(tmp_script))) as stream:
assert read_format_from_metadata(stream.read(), auto_ext) == 'percent'
# reload and compare with original notebook
model = cm.get(path=tmp_script)
compare_notebooks(nb, model['content'])
def test_sync_with_pre_commit_hook(tmpdir):
# Init git and create a pre-commit hook
git = git_in_tmpdir(tmpdir)
hook = str(tmpdir.join(".git/hooks/pre-commit"))
with open(hook, "w") as fp:
fp.write("#!/bin/sh\n" "jupytext --sync --pre-commit\n")
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
# Create a notebook that is not paired
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_md = str(tmpdir.join("notebook.md"))
nb = new_notebook(cells=[new_markdown_cell("A short notebook")])
write(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_md)
git("add", "notebook.ipynb")
git("status")
git("commit", "-m", "created")
git("status")
assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only")
assert "notebook.md" not in git("ls-tree", "-r", "master", "--name-only")
assert os.path.isfile(tmp_ipynb)
assert not os.path.exists(tmp_md)
# Pair the notebook
jupytext(["--set-formats", "ipynb,md", tmp_ipynb])
# Remove the md file (it will be regenerated by the pre-commit hook)
os.remove(tmp_md)
# Commit the ipynb file
git("add", "notebook.ipynb")
git("status")
git("commit", "-m", "paired")
git("status")
# The pre-commit script should have created and committed the md file
assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only")
assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only")
assert os.path.isfile(tmp_md)
nb_md = read(tmp_md)
compare_notebooks(nb_md, nb)
# Edit the md file
with open(tmp_md) as fp:
md_text = fp.read()
with open(tmp_md, "w") as fp:
fp.write(md_text.replace("A short notebook", "Notebook was edited"))
# commit the md file
git("add", "notebook.md")
git("status")
git("commit", "-m", "edited md")
git("status")
# The pre-commit script should have sync and committed the ipynb file
assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only")
assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only")
nb = read(tmp_ipynb)
compare(nb.cells, [new_markdown_cell("Notebook was edited")])
# create and commit a jpg file
tmp_jpg = str(tmpdir.join("image.jpg"))
with open(tmp_jpg, "wb") as fp:
fp.write(b"")
git("add", "image.jpg")
git("commit", "-m", "added image")
def test_as_version_has_appropriate_type():
with pytest.raises(TypeError):
jupytext.read("script.py", "py:percent")
def test_pre_commit_hook_sync_black_nbstripout(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout."""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --pipe, black]
additional_dependencies:
- black==20.8b1 # Matches hook
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
- repo: https://github.com/kynan/nbstripout
rev: 0.3.9
hooks:
- id: nbstripout
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
tmpdir.join(".jupytext.toml").write('default_jupytext_formats = "ipynb,py:percent"')
tmp_repo.git.add(".jupytext.toml")
tmp_repo.index.commit("pair notebooks")
# write a test notebook
write(notebook_with_outputs, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the first cell is '1+1' which is not black compliant
# 3. the notebook has outputs
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# the first cell was reformatted
nb = read("test.ipynb")
assert nb.cells[0].source == "1 + 1"
# the ipynb file has no outputs
assert not nb.cells[0].outputs
'func_render_all', 'func_render_instancing', 'func_accel_consistency',
'func_py_custom_material', 'func_py_custom_renderer',
'func_distributed_rendering', 'func_distributed_rendering_ext',
'func_error_handling', 'func_obj_loader_consistency',
'func_render_instancing', 'func_serial_consistency',
'func_update_asset', 'perf_accel', 'perf_obj_loader', 'perf_serial'
]
# Execute tests
for test in tests:
print(Fore.GREEN + "Running test [name='{}']".format(test) +
Style.RESET_ALL,
flush=True)
# Read the requested notebook
nb = jupytext.read(os.path.join(base_path, test + '.py'))
# Execute the notebook
ep = ExecutePreprocessor(timeout=600)
ep.preprocess(nb, {'metadata': {'path': base_path}})
# Write result
with open(os.path.join(args.output_dir, test + '.ipynb'),
mode='w',
encoding='utf-8') as f:
nbformat.write(nb, f)
# Merge executed notebooks
print(Fore.GREEN + "Merging notebooks" + Style.RESET_ALL)
notebook_paths = [
os.path.join(args.output_dir, test + '.ipynb') for test in tests
def test_read_wrong_ext(tmpdir, nb_file='notebook.ext'):
nb_file = tmpdir.join(nb_file)
nb_file.write('{}')
with pytest.raises(JupytextFormatError):
jupytext.read(str(nb_file))
def test_format_with_extension_change(tmp_nbs):
dag = DAGSpec('pipeline.yaml').to_dag().render()
dag['load'].source.format(fmt='ipynb')
assert not Path('load.py').exists()
assert jupytext.read('load.ipynb')
def convert_to_myst(file, dest_path):
# Read a notebook from a file
ntbk = jupytext.read(file, fmt='ipynb')
# Write notebook to md file
jupytext.write(ntbk, dest_path.joinpath(file.stem + '.md'), fmt='md')
def to_notebook(path):
notebook = jupytext.read(path)
jupytext.write(notebook, replace_extension(path, ".ipynb"))
def build_page(path_ntbk,
path_html_output,
path_media_output=None,
execute=False,
path_template=None,
verbose=False,
kernel_name=None):
"""Build the HTML for a single notebook page.
Inputs
======
path_ntbk : string
The path to a notebook or text file we want to convert. If a text
file, then Jupytext will be used to convert into a notebook. This
will also cause the notebook to be *run* (e.g. execute=True).
path_html_output : string
The path to the folder where the HTML will be output.
path_media_output : string | None
If a string, the path to where images should be extracted. If None,
images will be embedded in the HTML.
execute : bool
Whether to execute the notebook before converting
path_template : string
A path to the template used in conversion.
kernel_name : string
The name of the kernel to use if we execute notebooks.
"""
########################################
# Load in the notebook
notebook_name, suff = op.splitext(op.basename(path_ntbk))
is_raw_markdown_file = False
if suff in ['.md', '.markdown']:
# If it's a markdown file, we need to check whether it's a jupytext format
with open(path_ntbk, 'r') as ff:
lines = ff.readlines()
yaml_lines, content = _split_yaml(lines)
yaml = YAML().load(''.join(yaml_lines))
if (yaml is not None) and yaml.get('jupyter', {}).get('jupytext'):
# If we have jupytext metadata, then use it to read the markdown file
ntbk = jpt.reads(''.join(lines), 'md')
else:
# Otherwise, create an empty notebook and add all of the file contents as a markdown file
is_raw_markdown_file = True
ntbk = nbf.v4.new_notebook()
ntbk['cells'].append(
nbf.v4.new_markdown_cell(source=''.join(content)))
else:
# If it's not markdown, we assume it's either ipynb or a jupytext format
ntbk = jpt.read(path_ntbk)
if _is_jupytext_file(ntbk):
execute = True
########################################
# Notebook cleaning
# Minor edits to cells
_clean_markdown_cells(ntbk)
#############################################
# Conversion to HTML
# create a configuration object that changes the preprocessors
c = Config()
c.FilesWriter.build_directory = path_html_output
# Remove cell elements using tags
c.TagRemovePreprocessor.remove_cell_tags = ("remove_cell", "removecell")
c.TagRemovePreprocessor.remove_all_outputs_tags = ('remove_output', )
c.TagRemovePreprocessor.remove_input_tags = ('remove_input', )
# Remove any cells that are *only* whitespace
c.RegexRemovePreprocessor.patterns = ["\\s*\\Z"]
c.HTMLExporter.preprocessors = [
'nbconvert.preprocessors.TagRemovePreprocessor',
'nbconvert.preprocessors.RegexRemovePreprocessor',
# So the images are written to disk
'nbconvert.preprocessors.ExtractOutputPreprocessor',
# Wrap cells in Jekyll raw tags
_RawCellPreprocessor,
]
# The text used as the text for anchor links.
# TEMPORATILY Set to empty since we'll use anchor.js for the links
# Once https://github.com/jupyter/nbconvert/pull/1101 is fixed
# set to '<i class="fas fa-link"> </i>'
c.HTMLExporter.anchor_link_text = ' '
# Excluding input/output prompts
c.HTMLExporter.exclude_input_prompt = True
c.HTMLExporter.exclude_output_prompt = True
# Excution of the notebook if we wish
if execute is True:
ntbk = run_ntbk(ntbk, op.dirname(path_ntbk))
# Define the path to images and then the relative path to where they'll originally be placed
if isinstance(path_media_output, str):
path_media_output_rel = op.relpath(path_media_output, path_html_output)
# Generate HTML from our notebook using the template
output_resources = {
'output_files_dir': path_media_output_rel,
'unique_key': notebook_name
}
exp = HTMLExporter(template_file=path_template, config=c)
html, resources = exp.from_notebook_node(ntbk, resources=output_resources)
html = '<main class="jupyter-page">\n' + html + '\n</main>\n'
# Now write the markdown and resources
writer = FilesWriter(config=c)
writer.write(html, resources, notebook_name=notebook_name)
# Add the frontmatter to the yaml file in case it's wanted
if is_raw_markdown_file and len(yaml_lines) > 0:
with open(op.join(path_html_output, notebook_name + '.html'),
'r') as ff:
md_lines = ff.readlines()
md_lines.insert(0, '---\n')
for iline in yaml_lines[::-1]:
md_lines.insert(0, iline + '\n')
md_lines.insert(0, '---\n')
with open(op.join(path_html_output, notebook_name + '.html'),
'w') as ff:
ff.writelines(md_lines)
if verbose:
print("Finished writing notebook to {}".format(path_html_output))
def build_book(path_book,
path_toc_yaml=None,
path_ssg_config=None,
path_template=None,
local_build=False,
execute=False,
overwrite=False):
"""Build the HTML for a book using its TOC and a content folder.
Parameters
----------
path_book : str
Path to the root of the book repository
path_toc_yaml : str | None
Path to the Table of Contents YAML file
path_ssg_config : str | None
Path to the Jekyll configuration file
path_template : str | None
Path to the template nbconvert uses to build HTML
files
local_build : bool
Specify you are building site locally for later upload
execute : bool
Whether to execute notebooks before converting to HTML
overwrite : bool
Whether to overwrite existing HTML files
"""
if not op.isdir(path_book):
raise _error("Could not find a Jupyter Book at the given location.\n"
"Double-check the path you've provided:\n"
"\n"
f"{path_book}")
_check_book_versions(path_book)
PATH_IMAGES_FOLDER = op.join(path_book, '_build', 'images')
BUILD_FOLDER = op.join(path_book, BUILD_FOLDER_NAME)
###############################################
# Read in textbook configuration
# Load the yaml for this site
with open(path_ssg_config, 'r') as ff:
site_yaml = yaml.safe_load(ff.read())
CONTENT_FOLDER_NAME = site_yaml.get('content_folder_name').strip('/')
PATH_CONTENT_FOLDER = op.join(path_book, CONTENT_FOLDER_NAME)
# Load the textbook yaml for this site
if not op.exists(path_toc_yaml):
raise _error("No toc.yml file found, please create one at `{}`".format(
path_toc_yaml))
with open(path_toc_yaml, 'r') as ff:
toc = yaml.safe_load(ff.read())
# Drop divider items and non-linked pages in the sidebar, un-nest sections
toc = _prepare_toc(toc)
################################################
# Generating the Jekyll files for all content
n_skipped_files = 0
n_built_files = 0
case_check = _case_sensitive_fs(BUILD_FOLDER) and local_build
print("Convert and copy notebook/md files...")
for ix_file, page in enumerate(tqdm(list(toc))):
url_page = page.get('url', None)
title = page.get('title', None)
if page.get('external', None):
# If its an external link, just pass
continue
# Make sure URLs (file paths) have correct structure
_check_url_page(url_page, CONTENT_FOLDER_NAME)
##############################################
# Create path to old/new file and create directory
# URL will be relative to the CONTENT_FOLDER
path_url_page = os.path.join(PATH_CONTENT_FOLDER, url_page.lstrip('/'))
path_url_folder = os.path.dirname(path_url_page)
# URLs shouldn't have the suffix in there already so
# now we find which one to add
for suff in SUPPORTED_FILE_SUFFIXES:
if op.exists(path_url_page + suff):
path_url_page = path_url_page + suff
break
elif suff == "#BREAK#":
# Final suffix means we didn't find any existing content
raise _error(
"Could not find file called {} with any of these extensions: {}"
.format(path_url_page, SUPPORTED_FILE_SUFFIXES[:-1]))
# Create and check new folder / file paths
path_build_new_folder = path_url_folder.replace(
os.sep + CONTENT_FOLDER_NAME, os.sep + BUILD_FOLDER_NAME) + os.sep
path_build_new_file = op.join(
path_build_new_folder,
op.basename(path_url_page).replace(suff, '.html'))
# If the new build file exists and is *newer* than the original file, assume
# the original content file hasn't changed and skip it.
if overwrite is False and op.exists(path_build_new_file) \
and _file_newer_than(path_build_new_file, path_url_page):
n_skipped_files += 1
continue
if not op.isdir(path_build_new_folder):
os.makedirs(path_build_new_folder)
################################################
# Generate previous/next page URLs
if ix_file == 0:
url_prev_page = ''
prev_file_title = ''
else:
prev_file_title = toc[ix_file - 1].get('title')
url_prev_page = toc[ix_file - 1].get('url')
pre_external = toc[ix_file - 1].get('external', False)
if pre_external is False:
url_prev_page = _prepare_url(url_prev_page)
if ix_file == len(toc) - 1:
url_next_page = ''
next_file_title = ''
else:
next_file_title = toc[ix_file + 1].get('title')
url_next_page = toc[ix_file + 1].get('url')
next_external = toc[ix_file + 1].get('external', False)
if next_external is False:
url_next_page = _prepare_url(url_next_page)
###############################################################################
# Get kernel name and presence of widgets from notebooks metadata
kernel_name = ''
data = jpt.read(path_url_page)
if 'metadata' in data and 'kernelspec' in data['metadata']:
kernel_name = data['metadata']['kernelspec']['name']
has_widgets = "true" if any(
"interactive" in cell['metadata'].get('tags', [])
for cell in data['cells']) else "false"
############################################
# Content conversion
# Convert notebooks or just copy md if no notebook.
if any(path_url_page.endswith(ii) for ii in ['.md', '.ipynb']):
# Decide the path where the images will be placed, relative to the HTML location
path_after_build_folder = path_build_new_folder.split(
os.sep + BUILD_FOLDER_NAME + os.sep)[-1]
path_images_new_folder = op.join(PATH_IMAGES_FOLDER,
path_after_build_folder)
# Build the HTML for this book
build_page(path_url_page,
path_build_new_folder,
path_images_new_folder,
path_template=path_template,
kernel_name=kernel_name,
execute=execute)
else:
raise _error("Files must end in ipynb or md. Found file {}".format(
path_url_page))
###############################################################################
# Modify the generated HTML to work with the SSG
with open(path_build_new_file, 'r', encoding='utf8') as ff:
lines = ff.readlines()
# Split off original yaml
yaml_orig, lines = _split_yaml(lines)
# Front-matter YAML
yaml_fm = []
yaml_fm += ['---']
# In case pre-existing links are sanitized
sanitized = url_page.lower().replace('_', '-')
if sanitized != url_page:
if case_check and url_page.lower() == sanitized:
raise RuntimeError(
'Redirect {} clashes with page {} for local build on '
'case-insensitive FS\n'.format(sanitized, url_page) +
'Rename source page to lower case or build on a case '
'sensitive FS, e.g. case-sensitive disk image on Mac')
yaml_fm += ['redirect_from:']
yaml_fm += [' - "{}"'.format(sanitized)]
# Add interactive kernel info
interact_path = CONTENT_FOLDER_NAME + '/' + \
path_url_page.split(CONTENT_FOLDER_NAME + '/')[-1]
yaml_fm += ['interact_link: {}'.format(interact_path)]
yaml_fm += ["kernel_name: {}".format(kernel_name)]
yaml_fm += ["has_widgets: {}".format(has_widgets)]
# Page metadata
# Use YAML block scalars for titles so that people can use special characters
# See http://blogs.perl.org/users/tinita/2018/03/strings-in-yaml---to-quote-or-not-to-quote.html
yaml_fm += ["title: |-"]
yaml_fm += [" {}".format(title)]
yaml_fm += ['prev_page:']
yaml_fm += [' url: {}'.format(url_prev_page)]
yaml_fm += [" title: |-"]
yaml_fm += [" {}".format(prev_file_title)]
yaml_fm += ['next_page:']
yaml_fm += [' url: {}'.format(url_next_page)]
yaml_fm += [" title: |-"]
yaml_fm += [" {}".format(next_file_title)]
# Add back any original YaML, and end markers
yaml_fm += yaml_orig
yaml_fm += [
'comment: "***PROGRAMMATICALLY GENERATED, DO NOT EDIT. SEE ORIGINAL FILES IN /{}***"'
.format(CONTENT_FOLDER_NAME)
]
yaml_fm += ['---']
yaml_fm = [ii + '\n' for ii in yaml_fm]
lines = yaml_fm + lines
# Write the result as UTF-8.
with open(path_build_new_file, 'w', encoding='utf8') as ff:
ff.writelines(lines)
n_built_files += 1
#######################################################
# Finishing up...
# Copy non-markdown files in notebooks/ in case they're referenced in the notebooks
print('Copying non-content files inside `{}/`...'.format(
CONTENT_FOLDER_NAME))
_copy_non_content_files(PATH_CONTENT_FOLDER, CONTENT_FOLDER_NAME,
BUILD_FOLDER_NAME)
# Message at the end
msg = [
"Generated {} new files\nSkipped {} already-built files".format(
n_built_files, n_skipped_files)
]
if n_built_files == 0:
msg += [
"Delete the markdown/HTML files in '{}' for any pages that you wish to re-build, or use --overwrite option to re-build all."
.format(BUILD_FOLDER_NAME)
]
msg += ["Your Jupyter Book is now in `{}/`.".format(BUILD_FOLDER_NAME)]
msg += ["Demo your Jupyter book with `make serve` or push to GitHub!"]
print_message_box('\n'.join(msg))
