def test_sync_pandoc(nb_file, tmpdir, capsys):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_md = str(tmpdir.join('notebook.md'))
nb = read(nb_file)
write(nb, tmp_ipynb)
# Test that sync issues a warning when the notebook is not paired
jupytext(['--sync', tmp_ipynb])
_, err = capsys.readouterr()
assert 'is not a paired notebook' in err
# Now with a pairing information
nb.metadata.setdefault('jupytext', {})['formats'] = 'ipynb,md:pandoc'
write(nb, tmp_ipynb)
# Test that missing files are created
jupytext(['--sync', tmp_ipynb])
assert os.path.isfile(tmp_md)
compare_notebooks(nb, read(tmp_md), 'md:pandoc')
with open(tmp_md) as fp:
assert 'pandoc' in fp.read()
def test_sync_pandoc(nb_file, tmpdir, capsys):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_md = str(tmpdir.join("notebook.md"))
nb = read(nb_file)
write(nb, tmp_ipynb)
# Test that sync issues a warning when the notebook is not paired
jupytext(["--sync", tmp_ipynb])
_, err = capsys.readouterr()
assert "is not a paired notebook" in err
# Now with a pairing information
nb.metadata.setdefault("jupytext", {})["formats"] = "ipynb,md:pandoc"
write(nb, tmp_ipynb)
# Test that missing files are created
jupytext(["--sync", tmp_ipynb])
assert os.path.isfile(tmp_md)
compare_notebooks(read(tmp_md), nb, "md:pandoc")
with open(tmp_md) as fp:
assert "pandoc" in fp.read()
def render_rmd(input_file: str, output_file: str, params: dict = None):
"""
Wrapper function to render an Rmarkdown document with
the R `rmarkdown` package and convert it to HTML using pandoc
and a custom template.
Parameters
----------
input_file
path to input (Rmd) file
output_file
path to output (html) file
params
Dictionary that will be passed to `params` arg of `rmarkdown::render`.
See https://bookdown.org/yihui/rmarkdown/parameterized-reports.html for more details.
"""
# Directory the notebook is located in. Will be used as additional resource path for pandoc.
nb_dir = os.path.abspath(os.path.dirname(input_file))
with TemporaryDirectory() as tmp_dir:
with TemporaryDirectory() as tmp_dir_nb_converted:
# Create this file manually with given name
# so that pandoc gracefully falls back to the
# file name if no title is specified within the file. (#17)
tmp_nb_converted = os.path.join(
tmp_dir_nb_converted,
os.path.splitext(os.path.basename(input_file))[0] + ".Rmd",
)
if not input_file.endswith(".Rmd"):
nb = jtx.read(input_file)
jtx.write(nb, tmp_nb_converted)
else:
copyfile(input_file, tmp_nb_converted)
md_file = _run_rmarkdown(tmp_nb_converted, tmp_dir, params)
run_pandoc(md_file, output_file, res_path=[nb_dir, tmp_dir])
def test_pre_commit_hook_ipynb_to_py(
tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev
):
"""Here we document and test the expected behavior of the pre-commit hook in the
directional (--to) mode. Note that here, the ipynb file is always the source for
updates - i.e. changes on the .py file will not trigger the hook.
"""
# set up the tmpdir repo with pre-commit
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--from, ipynb, --to, "py:percent"]
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks"])
# write test notebook and output file
nb = new_notebook(cells=[new_markdown_cell("A short notebook")])
write(nb, "test.ipynb")
jupytext(["--from", "ipynb", "--to", "py:percent", "test.ipynb"])
tmp_repo.git.add(".")
tmp_repo.index.commit("test")
# make a change to the notebook
nb = new_notebook(cells=[new_markdown_cell("Some other text")])
write(nb, "test.ipynb")
tmp_repo.git.add("test.ipynb")
# now a commit will fail, and keep failing until we add the new
# changes made to the existing output to the index ourselves
with pytest.raises(HookExecutionError, match="files were modified by this hook"):
tmp_repo.index.commit("fails")
with pytest.raises(HookExecutionError, match="git add test.py"):
tmp_repo.index.commit("fails again")
# once we add the changes, it will pass
tmp_repo.git.add("test.py")
tmp_repo.index.commit("succeeds")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# Updating the .py file is possible
nb = new_notebook(cells=[new_markdown_cell("Some updated text")])
write(nb, "test.py", fmt="py:percent")
tmp_repo.index.commit("update py version")
# But it won't change the ipynb file (if you want that, use the --sync mode)
nb = read("test.ipynb")
assert nb.cells == [new_markdown_cell("Some other text")]
def test_sync_pipe_config(tmpdir):
"""Sync a notebook to a script paired in a tree, and reformat the markdown cells using pandoc"""
tmpdir.join("jupytext.toml").write(
"""# By default, the notebooks in this repository are in the notebooks subfolder
# and they are paired to scripts in the script subfolder.
default_jupytext_formats = "notebooks///ipynb,scripts///py:percent"
"""
)
nb_file = tmpdir.mkdir("notebooks").join("wrap_markdown.ipynb")
long_text = "This is a " + ("very " * 24) + "long sentence."
assert len(long_text) > 100
nb = new_notebook(cells=[new_markdown_cell(long_text)])
write(nb, str(nb_file))
jupytext(
[
"--sync",
"--pipe-fmt",
"ipynb",
"--pipe",
"pandoc --from ipynb --to ipynb --atx-headers",
str(nb_file),
]
)
py_text = tmpdir.join("scripts").join("wrap_markdown.py").read()
assert "This is a very very" in py_text
for line in py_text.splitlines():
assert len(line) <= 79
nb = read(nb_file, as_version=4)
text = nb.cells[0].source
assert len(text.splitlines()) == 3
assert text != long_text
def test_pre_commit_hook(tmpdir):
tmp_ipynb = str(tmpdir.join("nb with spaces.ipynb"))
tmp_py = str(tmpdir.join("nb with spaces.py"))
nb = new_notebook(cells=[])
git = git_in_tmpdir(tmpdir)
hook = str(tmpdir.join(".git/hooks/pre-commit"))
with open(hook, "w") as fp:
fp.write("#!/bin/sh\n" "jupytext --to py:light --pre-commit\n")
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
write(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_py)
git("add", "nb with spaces.ipynb")
git("status")
git("commit", "-m", "created")
git("status")
assert "nb with spaces.py" in git("ls-tree", "-r", "master", "--name-only")
assert os.path.isfile(tmp_py)
def test_combine_same_version_ok(tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_nbpy = str(tmpdir.join('notebook.py'))
tmp_rmd = str(tmpdir.join('notebook.Rmd'))
with open(tmp_nbpy, 'w') as fp:
fp.write("""# ---
# jupyter:
# jupytext_formats: ipynb,py
# jupytext_format_version: '1.2'
# ---
# New cell
""")
nb = new_notebook(metadata={'jupytext_formats': 'ipynb,py'})
write(nb, tmp_ipynb)
# to jupyter notebook
jupytext([tmp_nbpy, '--to', 'ipynb', '--update'])
# test round trip
jupytext([tmp_nbpy, '--to', 'notebook', '--test'])
# test ipynb to rmd
jupytext([tmp_ipynb, '--to', 'rmarkdown'])
nb = read(tmp_ipynb)
cells = nb['cells']
assert len(cells) == 1
assert cells[0].cell_type == 'markdown'
assert cells[0].source == 'New cell'
nb = read(tmp_rmd)
cells = nb['cells']
assert len(cells) == 1
assert cells[0].cell_type == 'markdown'
assert cells[0].source == 'New cell'
def test_combine_lower_version_raises(tmpdir):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_nbpy = str(tmpdir.join("notebook.py"))
with open(tmp_nbpy, "w") as fp:
fp.write("""# ---
# jupyter:
# jupytext:
# formats: ipynb,py:light
# text_representation:
# extension: .py
# format_name: light
# format_version: '55.4'
# jupytext_version: 42.1.1
# kernelspec:
# display_name: Python 3
# language: python
# name: python3
# ---
# New cell
""")
nb = new_notebook(metadata={"jupytext_formats": "ipynb,py"})
write(nb, tmp_ipynb)
with pytest.raises(
ValueError,
match=
"The file notebook.py was generated with jupytext version 42.1.1 but you have .* installed. "
"Please upgrade jupytext to version 42.1.1, or remove either notebook.py or notebook.ipynb. "
"This error occurs because notebook.py is in the light format in version 55.4, "
"while jupytext version .* installed at .* can only read the light format in versions .*",
):
jupytext([tmp_nbpy, "--to", "ipynb", "--update"])
def test_combine_same_version_ok(tmpdir):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_nbpy = str(tmpdir.join("notebook.py"))
tmp_rmd = str(tmpdir.join("notebook.Rmd"))
with open(tmp_nbpy, "w") as fp:
fp.write("""# ---
# jupyter:
# jupytext_formats: ipynb,py
# jupytext_format_version: '1.2'
# ---
# New cell
""")
nb = new_notebook(metadata={"jupytext_formats": "ipynb,py"})
write(nb, tmp_ipynb)
# to jupyter notebook
jupytext([tmp_nbpy, "--to", "ipynb", "--update"])
# test round trip
jupytext([tmp_nbpy, "--to", "notebook", "--test"])
# test ipynb to rmd
jupytext([tmp_ipynb, "--to", "rmarkdown"])
nb = read(tmp_ipynb)
cells = nb["cells"]
assert len(cells) == 1
assert cells[0].cell_type == "markdown"
assert cells[0].source == "New cell"
nb = read(tmp_rmd)
cells = nb["cells"]
assert len(cells) == 1
assert cells[0].cell_type == "markdown"
assert cells[0].source == "New cell"
def test_manual_call_of_pre_commit_hook(tmpdir):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_py = str(tmpdir.join("notebook.py"))
nb = new_notebook(cells=[])
os.chdir(str(tmpdir))
git = git_in_tmpdir(tmpdir)
def hook():
with mock.patch("jupytext.cli.system", system_in_tmpdir(tmpdir)):
jupytext(["--to", "py", "--pre-commit"])
write(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_py)
git("add", "notebook.ipynb")
git("status")
hook()
git("commit", "-m", "created")
git("status")
assert "notebook.py" in git("ls-tree", "-r", "master", "--name-only")
assert os.path.isfile(tmp_py)
def test_rst2md(tmpdir):
tmp_py = str(tmpdir.join('notebook.py'))
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
# Write notebook in sphinx format
nb = new_notebook(cells=[
new_markdown_cell('A short sphinx notebook'),
new_markdown_cell(':math:`1+1`')
])
write(nb, tmp_py, fmt='py:sphinx')
jupytext([
tmp_py, '--from', 'py:sphinx', '--to', 'ipynb', '--opt', 'rst2md=True',
'--opt', 'cell_metadata_filter=-all'
])
assert os.path.isfile(tmp_ipynb)
nb = read(tmp_ipynb)
assert nb.metadata['jupytext']['cell_metadata_filter'] == '-all'
assert nb.metadata['jupytext']['rst2md'] is False
# Was rst to md conversion effective?
assert nb.cells[2].source == '$1+1$'
def test_apply_black_through_jupytext(tmpdir, nb_file):
# Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work
metadata = read(nb_file).metadata
nb_org = new_notebook(cells=[new_code_cell("1 +1")], metadata=metadata)
nb_black = new_notebook(cells=[new_code_cell("1 + 1")], metadata=metadata)
os.makedirs(str(tmpdir.join("notebook_folder")))
os.makedirs(str(tmpdir.join("script_folder")))
tmp_ipynb = str(tmpdir.join("notebook_folder").join("notebook.ipynb"))
tmp_py = str(tmpdir.join("script_folder").join("notebook.py"))
# Black in place
write(nb_org, tmp_ipynb)
jupytext([tmp_ipynb, "--pipe", "black"])
nb_now = read(tmp_ipynb)
compare(nb_now, nb_black)
# Write to another folder using dots
script_fmt = os.path.join("..", "script_folder//py:percent")
write(nb_org, tmp_ipynb)
jupytext([tmp_ipynb, "--to", script_fmt, "--pipe", "black"])
assert os.path.isfile(tmp_py)
nb_now = read(tmp_py)
nb_now.metadata = metadata
compare(nb_now, nb_black)
os.remove(tmp_py)
# Map to another folder based on file name
write(nb_org, tmp_ipynb)
jupytext(
[
tmp_ipynb,
"--from",
"notebook_folder//ipynb",
"--to",
"script_folder//py:percent",
"--pipe",
"black",
"--check",
"flake8",
]
)
assert os.path.isfile(tmp_py)
nb_now = read(tmp_py)
nb_now.metadata = metadata
compare(nb_now, nb_black)
def test_format_prefix_suffix(tmpdir):
os.makedirs(str(tmpdir.join("notebooks")))
tmp_ipynb = str(tmpdir.join("notebooks/notebook_name.ipynb"))
tmp_py = str(tmpdir.join("scripts/notebook_name.py"))
write(new_notebook(), tmp_ipynb)
jupytext([tmp_ipynb, "--to", os.path.join("..", "scripts//py")])
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
jupytext([tmp_ipynb, "--to", "scripts//py", "--from", "notebooks//ipynb"])
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
tmp_ipynb = str(tmpdir.join("notebooks/nb_prefix_notebook_name.ipynb"))
tmp_py = str(tmpdir.join("scripts/script_prefix_notebook_name.py"))
write(new_notebook(), tmp_ipynb)
jupytext(
[
tmp_ipynb,
"--to",
"scripts/script_prefix_/py",
"--from",
"notebooks/nb_prefix_/ipynb",
]
)
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
tmp_ipynb = str(tmpdir.join("notebooks/nb_prefix_notebook_name_nb_suffix.ipynb"))
tmp_py = str(tmpdir.join("scripts/script_prefix_notebook_name_script_suffix.py"))
write(new_notebook(), tmp_ipynb)
jupytext(
[
tmp_ipynb,
"--to",
"scripts/script_prefix_/_script_suffix.py",
"--from",
"notebooks/nb_prefix_/_nb_suffix.ipynb",
]
)
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
def test_apply_black_through_jupytext(tmpdir, nb_file):
# Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work
metadata = read(nb_file).metadata
nb_org = new_notebook(cells=[new_code_cell('1 +1')],
metadata=metadata)
nb_black = new_notebook(cells=[new_code_cell('1 + 1')], metadata=metadata)
os.makedirs(str(tmpdir.join('notebook_folder')))
os.makedirs(str(tmpdir.join('script_folder')))
tmp_ipynb = str(tmpdir.join('notebook_folder').join('notebook.ipynb'))
tmp_py = str(tmpdir.join('script_folder').join('notebook.py'))
# Black in place
write(nb_org, tmp_ipynb)
jupytext([tmp_ipynb, '--pipe', 'black'])
nb_now = read(tmp_ipynb)
compare(nb_now, nb_black)
# Write to another folder using dots
script_fmt = os.path.join('..', 'script_folder//py:percent')
write(nb_org, tmp_ipynb)
jupytext([tmp_ipynb, '--to', script_fmt, '--pipe', 'black'])
assert os.path.isfile(tmp_py)
nb_now = read(tmp_py)
nb_now.metadata = metadata
compare(nb_now, nb_black)
os.remove(tmp_py)
# Map to another folder based on file name
write(nb_org, tmp_ipynb)
jupytext([
tmp_ipynb, '--from', 'notebook_folder//ipynb', '--to',
'script_folder//py:percent', '--pipe', 'black', '--check', 'flake8'
])
assert os.path.isfile(tmp_py)
nb_now = read(tmp_py)
nb_now.metadata = metadata
compare(nb_now, nb_black)
def test_format_prefix_suffix(tmpdir):
os.makedirs(str(tmpdir.join('notebooks')))
tmp_ipynb = str(tmpdir.join('notebooks/notebook_name.ipynb'))
tmp_py = str(tmpdir.join('scripts/notebook_name.py'))
write(new_notebook(), tmp_ipynb)
jupytext([tmp_ipynb, '--to', os.path.join('..', 'scripts//py')])
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
jupytext([tmp_ipynb, '--to', 'scripts//py', '--from', 'notebooks//ipynb'])
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
tmp_ipynb = str(tmpdir.join('notebooks/nb_prefix_notebook_name.ipynb'))
tmp_py = str(tmpdir.join('scripts/script_prefix_notebook_name.py'))
write(new_notebook(), tmp_ipynb)
jupytext([
tmp_ipynb, '--to', 'scripts/script_prefix_/py', '--from',
'notebooks/nb_prefix_/ipynb'
])
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
tmp_ipynb = str(
tmpdir.join('notebooks/nb_prefix_notebook_name_nb_suffix.ipynb'))
tmp_py = str(
tmpdir.join('scripts/script_prefix_notebook_name_script_suffix.py'))
write(new_notebook(), tmp_ipynb)
jupytext([
tmp_ipynb, '--to', 'scripts/script_prefix_/_script_suffix.py',
'--from', 'notebooks/nb_prefix_/_nb_suffix.ipynb'
])
assert os.path.isfile(tmp_py)
os.remove(tmp_py)
def test_pre_commit_hook_sync_reformat_code_and_markdown(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and also apply black and pandoc to reformat both
code and markdown cells.
Note: the new cell ids introduced in nbformat 5.1.0 are not yet supported by all the programs that treat
ipynb files. Consequently we pin the version of nbformat to 5.0.8 in all the environments below (and you
will have to do the same on the environment in which you edit the notebooks).
"""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --pipe-fmt, ipynb, --pipe, 'pandoc --from ipynb --to ipynb --markdown-headings=atx', --diff]
additional_dependencies:
- nbformat==5.0.8 # because pandoc 2.11.4 does not preserve yet the new cell ids
- id: jupytext
args: [--sync, --pipe, black, --diff]
additional_dependencies:
- black==20.8b1 # Matches black hook below
- nbformat==5.0.8 # for compatibility with the pandoc hook above
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
tmpdir.join(".jupytext.toml").write('formats = "ipynb,py:percent"')
tmp_repo.git.add(".jupytext.toml")
tmp_repo.index.commit("pair notebooks")
# write a test notebook
notebook = new_notebook(
cells=[
new_code_cell("1+1"),
new_markdown_cell("""This is a complex markdown cell
# With a h1 header
## And a h2 header
| And | A | Table |
| --------- | ---| ----- |
| 0 | 1 | 2 |
!!!WARNING!!! This hook does not seem compatible with
explicit paragraph breaks (two spaces at the end of a line).
And a VERY long line.
""".replace("VERY ", "very " * 51)),
],
metadata=notebook_with_outputs.metadata,
)
# We write the notebook in version 4.4, i.e. with the equivalent of nbformat version 5.0.8
notebook.nbformat = 4
notebook.nbformat_minor = 4
write(notebook, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the first cell is '1+1' which is not black compliant
# 3. the second cell needs to be wrapped
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# both the code and the markdown cells were reformatted
nb = read("test.ipynb")
assert nb.cells[0].source == "1 + 1"
print(nb.cells[1].source)
assert (nb.cells[1].source == """This is a complex markdown cell
# With a h1 header
## And a h2 header
| And | A | Table |
|-----|-----|-------|
| 0 | 1 | 2 |
!!!WARNING!!! This hook does not seem compatible with explicit paragraph
breaks (two spaces at the end of a line).
And a very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very long line.""")
def test_write_non_ascii(tmpdir):
nb = jupytext.reads(u"Non-ascii contênt", "Rmd")
jupytext.write(nb, str(tmpdir.join("notebook.Rmd")))
jupytext.write(nb, str(tmpdir.join("notebook.ipynb")))
def create_mirror_file_if_missing(mirror_file, notebook, fmt):
if not os.path.isfile(mirror_file):
jupytext.write(notebook, mirror_file, fmt=fmt)
def tmp_ipynb(tmpdir):
tmp_file = str(tmpdir.join("notebook.ipynb"))
write(new_notebook(), tmp_file)
return tmp_file
def test_pre_commit_hook_sync_black_nbstripout(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout."""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --pipe, black]
additional_dependencies:
- black==20.8b1 # Matches hook
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
- repo: https://github.com/kynan/nbstripout
rev: 0.3.9
hooks:
- id: nbstripout
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
tmpdir.join(".jupytext.toml").write('default_jupytext_formats = "ipynb,py:percent"')
tmp_repo.git.add(".jupytext.toml")
tmp_repo.index.commit("pair notebooks")
# write a test notebook
write(notebook_with_outputs, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the first cell is '1+1' which is not black compliant
# 3. the notebook has outputs
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# the first cell was reformatted
nb = read("test.ipynb")
assert nb.cells[0].source == "1 + 1"
# the ipynb file has no outputs
assert not nb.cells[0].outputs
def test_meaningfull_error_write_myst_missing(tmpdir):
nb_file = tmpdir.join("notebook.ipynb")
jupytext.write(new_notebook(), str(nb_file))
with pytest.raises(ImportError, match=PLEASE_INSTALL_MYST):
jupytext_cli([str(nb_file), "--to", "md:myst"])
def test_write_wrong_ext(tmpdir, nb_file='notebook.ext'):
nb_file = str(tmpdir.join(nb_file))
with pytest.raises(JupytextFormatError):
jupytext.write(new_notebook(), nb_file)
def develop(self, app='notebook', args=None):
"""
Opens the rendered notebook (with injected parameters) and adds a
"debugging-settings" cell to the that changes directory to the current
active directory. This will reflect conditions when callign
`DAG.build()`. This modified notebook is saved in the same location as
the source with a "-tmp" added to the filename. Changes to this
notebook can be exported to the original notebook after the notebook
process is shut down. The "injected-parameters" and
"debugging-settings" cells are deleted before saving.
Parameters
----------
app : {'notebook', 'lab'}, default: 'notebook'
Which Jupyter application to use
args : str
Extra parameters passed to the jupyter application
Notes
-----
Be careful when developing tasks interacively. If the task has run
successfully, you overwrite products but don't save the
updated source code, your DAG will enter an inconsistent state where
the metadata won't match the overwritten product.
If you modify the source code and call develop again, the source
code will be updated only if the ``hot_reload option`` is turned on.
See :class:`ploomber.DAGConfigurator` for details.
"""
# TODO: this code needs refactoring, should be a context manager
# like the one we have for PythonCallable.develop that abstracts
# the handling of the temporary notebook while editing
apps = {'notebook', 'lab'}
if app not in apps:
raise ValueError('"app" must be one of {}, got: "{}"'.format(
apps, app))
if self.source.language != 'python':
raise NotImplementedError(
'develop is not implemented for "{}" '
'notebooks, only python is supported'.format(
self.source.language))
if self.source.loc is None:
raise ValueError('Can only use develop in notebooks loaded '
'from files, not from str')
nb = _read_rendered_notebook(self.source.nb_str_rendered)
name = self.source.loc.name
suffix = self.source.loc.suffix
name_new = name.replace(suffix, '-tmp.ipynb')
tmp = self.source.loc.with_name(name_new)
content = nbformat.writes(nb, version=nbformat.NO_CONVERT)
tmp.write_text(content)
# open notebook with injected debugging cell
try:
subprocess.run(['jupyter', app, str(tmp)] +
shlex.split(args or ''),
check=True)
except KeyboardInterrupt:
print(f'Jupyter {app} application closed...')
# read tmp file again, to see if the user made any changes
content_new = Path(tmp).read_text()
# maybe exclude changes in tmp cells?
if content == content_new:
print('No changes found...')
else:
# save changes
if _save():
nb = nbformat.reads(content_new,
as_version=nbformat.NO_CONVERT)
# remove injected-parameters and debugging-settings cells if
# they exist
_cleanup_rendered_nb(nb)
# write back in the same format and original location
ext_source = Path(self.source.loc).suffix[1:]
print('Saving notebook to: ', self.source.loc)
jupytext.write(nb, self.source.loc, fmt=ext_source)
else:
print('Not saving changes...')
# remove tmp file
Path(tmp).unlink()
def convert_to_myst(file, dest_path):
# Read a notebook from a file
ntbk = jupytext.read(file, fmt='ipynb')
# Write notebook to md file
jupytext.write(ntbk, dest_path.joinpath(file.stem + '.md'), fmt='md')
def test_sync_with_pre_commit_hook(tmpdir):
# Init git and create a pre-commit hook
git = git_in_tmpdir(tmpdir)
git('init')
git('status')
hook = str(tmpdir.join('.git/hooks/pre-commit'))
with open(hook, 'w') as fp:
fp.write('#!/bin/sh\n' 'jupytext --sync --pre-commit\n')
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
# Create a notebook that is not paired
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_md = str(tmpdir.join('notebook.md'))
nb = new_notebook(cells=[new_markdown_cell('A short notebook')])
write(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_md)
git('add', 'notebook.ipynb')
git('status')
git('commit', '-m', 'created')
git('status')
assert 'notebook.ipynb' in git('ls-tree', '-r', 'master', '--name-only')
assert 'notebook.md' not in git('ls-tree', '-r', 'master', '--name-only')
assert os.path.isfile(tmp_ipynb)
assert not os.path.exists(tmp_md)
# Pair the notebook
jupytext(['--set-formats', 'ipynb,md', tmp_ipynb])
# Remove the md file (it will be regenerated by the pre-commit hook)
os.remove(tmp_md)
# Commit the ipynb file
git('add', 'notebook.ipynb')
git('status')
git('commit', '-m', 'paired')
git('status')
# The pre-commit script should have created and committed the md file
assert 'notebook.ipynb' in git('ls-tree', '-r', 'master', '--name-only')
assert 'notebook.md' in git('ls-tree', '-r', 'master', '--name-only')
assert os.path.isfile(tmp_md)
nb_md = read(tmp_md)
compare_notebooks(nb_md, nb)
# Edit the md file
with open(tmp_md) as fp:
md_text = fp.read()
with open(tmp_md, 'w') as fp:
fp.write(md_text.replace('A short notebook', 'Notebook was edited'))
# commit the md file
git('add', 'notebook.md')
git('status')
git('commit', '-m', 'edited md')
git('status')
# The pre-commit script should have sync and committed the ipynb file
assert 'notebook.ipynb' in git('ls-tree', '-r', 'master', '--name-only')
assert 'notebook.md' in git('ls-tree', '-r', 'master', '--name-only')
nb = read(tmp_ipynb)
compare(nb.cells, [new_markdown_cell('Notebook was edited')])
import os
import shutil
import jupytext
from tools import send_feedback
submission_dir = "/shared/submission/"
# submission_dir = "sub2/"
for file in os.listdir(submission_dir):
if file.endswith('.ipynb'):
learner_notebook = file
else:
learner_notebook = None
if learner_notebook is None:
send_feedback(0.0, "No notebook was found in the submission directory.")
exit()
sub_source = submission_dir + learner_notebook
sub_destination = '/grader/submission/submission.ipynb'
# sub_destination = 'submission/submission.ipynb'
shutil.copyfile(sub_source, sub_destination)
nb = jupytext.read("submission/submission.ipynb")
jupytext.write(nb, 'submission/submission.py', fmt='py:percent')
path_loader=data_path,
path_rf=data_path,
path_gabor=data_path,
)
path_output.mkdir(parents=True, exist_ok=True)
for nb in nbs:
try:
ntb = jupytext.read(nb + ".py")
ntb.metadata["kernelspec"] = {
"display_name": "Python 3",
"language": "python",
"name": "python3",
}
jupytext.write(ntb, nb + ".ipynb")
print(nb)
pm.execute_notebook(
f"{nb}.ipynb",
output_path=(path_output / f"{nb}_output.ipynb").as_posix(),
parameters=parameters,
)
except pm.exceptions.PapermillExecutionError as e:
logging.error(f"Error at {nb}.")
raise e
finally:
Path(nb + ".ipynb").unlink(missing_ok=True)
def test_sync_with_pre_commit_hook(tmpdir):
# Init git and create a pre-commit hook
git = git_in_tmpdir(tmpdir)
hook = str(tmpdir.join(".git/hooks/pre-commit"))
with open(hook, "w") as fp:
fp.write("#!/bin/sh\n" "jupytext --sync --pre-commit\n")
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
# Create a notebook that is not paired
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
tmp_md = str(tmpdir.join("notebook.md"))
nb = new_notebook(cells=[new_markdown_cell("A short notebook")])
write(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_md)
git("add", "notebook.ipynb")
git("status")
git("commit", "-m", "created")
git("status")
assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only")
assert "notebook.md" not in git("ls-tree", "-r", "master", "--name-only")
assert os.path.isfile(tmp_ipynb)
assert not os.path.exists(tmp_md)
# Pair the notebook
jupytext(["--set-formats", "ipynb,md", tmp_ipynb])
# Remove the md file (it will be regenerated by the pre-commit hook)
os.remove(tmp_md)
# Commit the ipynb file
git("add", "notebook.ipynb")
git("status")
git("commit", "-m", "paired")
git("status")
# The pre-commit script should have created and committed the md file
assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only")
assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only")
assert os.path.isfile(tmp_md)
nb_md = read(tmp_md)
compare_notebooks(nb_md, nb)
# Edit the md file
with open(tmp_md) as fp:
md_text = fp.read()
with open(tmp_md, "w") as fp:
fp.write(md_text.replace("A short notebook", "Notebook was edited"))
# commit the md file
git("add", "notebook.md")
git("status")
git("commit", "-m", "edited md")
git("status")
# The pre-commit script should have sync and committed the ipynb file
assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only")
assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only")
nb = read(tmp_ipynb)
compare(nb.cells, [new_markdown_cell("Notebook was edited")])
# create and commit a jpg file
tmp_jpg = str(tmpdir.join("image.jpg"))
with open(tmp_jpg, "wb") as fp:
fp.write(b"")
git("add", "image.jpg")
git("commit", "-m", "added image")
def to_notebook(path):
notebook = jupytext.read(path)
jupytext.write(notebook, replace_extension(path, ".ipynb"))
k2: when_first_appear(argsasadded, k2)
for k2 in parsingparser_ARGS.keys()
}
# add args line-by-line to the python file we are building in-memory
for k2 in sorted(list(keys2whenapp.keys()),
key=lambda x2: keys2whenapp[x2]):
assert ' ' not in k2 and '\t' not in k2 and '\n' not in k2 and '\r' not in k2, k2
dumpval = json.dumps(
parsingparser_ARGS[k2]
) # TODO: how to properly reproduce python CODE declaration of variable
if isinstance(parsingparser_ARGS[k2], bool):
dumpval = str(parsingparser_ARGS[k2])
FINALPRINTED += "args." + k2 + " = " + dumpval + "\n"
no_category_match = False
# if no argparse line category match, just print the line
# although, one last conversion fix: replace __file__ with the file path, since jupyter doesnt know __file__
if no_category_match:
line = join_tokens(line)
line = line.replace(
'(__file__)',
'(' + json.dumps(os.path.abspath(args.infile)) + ')')
if args.eval_until_argparse and parsingparser_ARGS is None:
evalline = line.strip('\r\n\t ')
if len(evalline) > 0 and evalline[0] != '#':
exec(evalline)
FINALPRINTED += line + "\n"
jupytext.write(jupytext.reads(FINALPRINTED, fmt='py'), args.out_dest)
