def test_pre_commit_hook_ipynb_to_py(tmpdir, cwd_tmpdir, tmp_repo,
jupytext_repo_root, jupytext_repo_rev):
"""Here we document and test the expected behavior of the pre-commit hook in the
directional (--to) mode. Note that here, the ipynb file is always the source for
updates - i.e. changes on the .py file will not trigger the hook.
"""
# set up the tmpdir repo with pre-commit
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--from, ipynb, --to, "py:percent"]
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks"])
# write test notebook and output file
nb = new_notebook(cells=[new_markdown_cell("A short notebook")])
write(nb, "test.ipynb")
jupytext(["--from", "ipynb", "--to", "py:percent", "test.ipynb"])
tmp_repo.git.add(".")
tmp_repo.index.commit("test")
# make a change to the notebook
nb = new_notebook(cells=[new_markdown_cell("Some other text")])
write(nb, "test.ipynb")
tmp_repo.git.add("test.ipynb")
# now a commit will fail, and keep failing until we add the new
# changes made to the existing output to the index ourselves
with pytest.raises(HookExecutionError,
match="files were modified by this hook"):
tmp_repo.index.commit("fails")
with pytest.raises(HookExecutionError, match="git add test.py"):
tmp_repo.index.commit("fails again")
# once we add the changes, it will pass
tmp_repo.git.add("test.py")
tmp_repo.index.commit("succeeds")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# Updating the .py file is possible
nb = new_notebook(cells=[new_markdown_cell("Some updated text")])
write(nb, "test.py", fmt="py:percent")
tmp_repo.index.commit("update py version")
# But it won't change the ipynb file (if you want that, use the --sync mode)
nb = read("test.ipynb")
compare_cells(nb.cells, [new_markdown_cell("Some other text")],
compare_ids=False)
def test_pre_commit_hook_sync_nbstripout(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a Markdown file and also apply nbstripout."""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync]
- repo: https://github.com/kynan/nbstripout
rev: 0.3.9
hooks:
- id: nbstripout
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
# write a test notebook
write(notebook_with_outputs, "test.ipynb")
# We pair the notebook to a md file
jupytext(["--set-formats", "ipynb,md", "test.ipynb"])
# try to commit it, should fail because
# 1. the md version hasn't been added
# 2. the notebook has outputs
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.md")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.md" in tmp_repo.tree()
# the ipynb file has no outputs on disk
nb = read("test.ipynb")
assert not nb.cells[0].outputs
def pre_commit_hooks() -> None:
""" Sets up git hooks """
try:
get_git_dir()
pre_commit(['install'])
except CalledProcessError:
log.warning(
'Git folder has not bee found! Proceed without git hooks setup step'
)
def test_pre_commit_hook_sync_execute(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and execute it (this is probably not a very
recommendable hook!)"""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --execute, --diff]
additional_dependencies:
- nbconvert
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
# simple notebook with kernel 'user_kernel_python3'
nb = notebook_with_outputs
nb.cells = [new_code_cell("3+4")]
# pair it to a py file and write it to disk
nb.metadata["jupytext"] = {"formats": "ipynb,py:percent"}
write(nb, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the outputs are missing
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# the first cell has the expected output
nb = read("test.ipynb")
assert nb.cells[0].outputs[0]["data"]["text/plain"] == "7"
def test_pre_commit_hook_for_existing_changed_file(tmpdir):
# get the path and revision of this repo, to use with pre-commit
repo_root = str(Path(__file__).parent.parent.resolve())
repo_rev = system("git", "rev-parse", "HEAD", cwd=repo_root).strip()
git = git_in_tmpdir(tmpdir)
# set up the tmpdir repo with pre-commit
pre_commit_config_yaml = dedent(f"""
repos:
- repo: {repo_root}
rev: {repo_rev}
hooks:
- id: jupytext
args: [--from, ipynb, --to, "py:light"]
""")
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
git("add", ".pre-commit-config.yaml")
with tmpdir.as_cwd():
pre_commit(["install", "--install-hooks"])
# write test notebook and output file
nb = new_notebook(cells=[new_markdown_cell("A short notebook")])
nb_file = str(tmpdir.join("test.ipynb"))
write(nb, nb_file)
py_file = str(tmpdir.join("test.py"))
jupytext(["--from", "ipynb", "--to", "py:light", str(nb_file)])
git("add", ".")
git("commit", "-m", "test")
# make a change to the notebook
nb = new_notebook(cells=[new_markdown_cell("Some other text")])
write(nb, nb_file)
git("add", nb_file)
# now a commit will fail, and keep failing until we add the new
# changes made to the existing output to the index ourselves
with pytest.raises(SystemExit):
git("commit", "-m", "fails")
with pytest.raises(SystemExit):
git("commit", "-m", "fails again")
# once we add the changes, it will pass
git("add", py_file)
git("commit", "-m", "succeeds")
assert "test.ipynb" in git("ls-files")
assert "test.py" in git("ls-files")
def test_pre_commit_hook_for_new_file(tmpdir):
# get the path and revision of this repo, to use with pre-commit
repo_root = str(Path(__file__).parent.parent.resolve())
repo_rev = system("git", "rev-parse", "HEAD", cwd=repo_root).strip()
# set up the tmpdir repo with pre-commit
git = git_in_tmpdir(tmpdir)
pre_commit_config_yaml = dedent(f"""
repos:
- repo: {repo_root}
rev: {repo_rev}
hooks:
- id: jupytext
args: [--sync]
""")
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
git("add", ".pre-commit-config.yaml")
with tmpdir.as_cwd():
pre_commit(["install", "--install-hooks", "-f"])
# write test notebook and sync it to py:percent
nb = new_notebook(cells=[new_markdown_cell("A short notebook")])
nb_file = str(tmpdir.join("test.ipynb"))
py_file = str(tmpdir.join("test.py"))
write(nb, nb_file)
with tmpdir.as_cwd():
jupytext(["--set-formats", "ipynb,py:percent", nb_file])
# try to commit it, should fail since the hook runs and makes changes
git("add", nb_file)
with pytest.raises(SystemExit):
git("commit", "-m", "failing")
# try again, it will still fail because the output hasn't been added
with pytest.raises(SystemExit):
git("commit", "-m", "still failing")
# add the new file, now the commit will succeed
git("add", py_file)
git("commit", "-m", "passing")
assert "test.ipynb" in git("ls-files")
assert "test.py" in git("ls-files")
def test_pre_commit_hook_sync_black_nbstripout(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout."""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --pipe, black]
additional_dependencies:
- black==20.8b1 # Matches hook
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
- repo: https://github.com/kynan/nbstripout
rev: 0.3.9
hooks:
- id: nbstripout
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
tmpdir.join(".jupytext.toml").write('default_jupytext_formats = "ipynb,py:percent"')
tmp_repo.git.add(".jupytext.toml")
tmp_repo.index.commit("pair notebooks")
# write a test notebook
write(notebook_with_outputs, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the first cell is '1+1' which is not black compliant
# 3. the notebook has outputs
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# the first cell was reformatted
nb = read("test.ipynb")
assert nb.cells[0].source == "1 + 1"
# the ipynb file has no outputs
assert not nb.cells[0].outputs
def test_pre_commit_hook_sync_reformat_code_and_markdown(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and also apply black and pandoc to reformat both
code and markdown cells.
Note: the new cell ids introduced in nbformat 5.1.0 are not yet supported by all the programs that treat
ipynb files. Consequently we pin the version of nbformat to 5.0.8 in all the environments below (and you
will have to do the same on the environment in which you edit the notebooks).
"""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --pipe-fmt, ipynb, --pipe, 'pandoc --from ipynb --to ipynb --markdown-headings=atx', --diff]
additional_dependencies:
- nbformat==5.0.8 # because pandoc 2.11.4 does not preserve yet the new cell ids
- id: jupytext
args: [--sync, --pipe, black, --diff]
additional_dependencies:
- black==20.8b1 # Matches black hook below
- nbformat==5.0.8 # for compatibility with the pandoc hook above
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
tmpdir.join(".jupytext.toml").write('formats = "ipynb,py:percent"')
tmp_repo.git.add(".jupytext.toml")
tmp_repo.index.commit("pair notebooks")
# write a test notebook
notebook = new_notebook(
cells=[
new_code_cell("1+1"),
new_markdown_cell("""This is a complex markdown cell
# With a h1 header
## And a h2 header
| And | A | Table |
| --------- | ---| ----- |
| 0 | 1 | 2 |
!!!WARNING!!! This hook does not seem compatible with
explicit paragraph breaks (two spaces at the end of a line).
And a VERY long line.
""".replace("VERY ", "very " * 51)),
],
metadata=notebook_with_outputs.metadata,
)
# We write the notebook in version 4.4, i.e. with the equivalent of nbformat version 5.0.8
notebook.nbformat = 4
notebook.nbformat_minor = 4
write(notebook, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the first cell is '1+1' which is not black compliant
# 3. the second cell needs to be wrapped
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# both the code and the markdown cells were reformatted
nb = read("test.ipynb")
assert nb.cells[0].source == "1 + 1"
print(nb.cells[1].source)
assert (nb.cells[1].source == """This is a complex markdown cell
# With a h1 header
## And a h2 header
| And | A | Table |
|-----|-----|-------|
| 0 | 1 | 2 |
!!!WARNING!!! This hook does not seem compatible with explicit paragraph
breaks (two spaces at the end of a line).
And a very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very long line.""")
def test_pre_commit_hook_sync_reformat_code_and_markdown(
tmpdir,
cwd_tmpdir,
tmp_repo,
jupytext_repo_root,
jupytext_repo_rev,
notebook_with_outputs,
):
"""Here we sync the ipynb notebook with a py:percent file and also apply black and pandoc to reformat both
code and markdown cells."""
pre_commit_config_yaml = f"""
repos:
- repo: {jupytext_repo_root}
rev: {jupytext_repo_rev}
hooks:
- id: jupytext
args: [--sync, --pipe-fmt, ipynb, --pipe, 'pandoc --from ipynb --to ipynb --markdown-headings=atx', --diff]
- id: jupytext
args: [--sync, --pipe, black, --diff]
additional_dependencies:
- black==20.8b1 # Matches hook
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
"""
tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml)
tmp_repo.git.add(".pre-commit-config.yaml")
pre_commit(["install", "--install-hooks", "-f"])
tmpdir.join(".jupytext.toml").write(
'default_jupytext_formats = "ipynb,py:percent"')
tmp_repo.git.add(".jupytext.toml")
tmp_repo.index.commit("pair notebooks")
# write a test notebook
notebook = new_notebook(
cells=[
new_code_cell("1+1"),
new_markdown_cell("""This is a complex markdown cell
# With a h1 header
## And a h2 header
| And | A | Table |
| --------- | ---| ----- |
| 0 | 1 | 2 |
!!!WARNING!!! This hook does not seem compatible with
explicit paragraph breaks (two spaces at the end of a line).
And a VERY long line.
""".replace("VERY ", "very " * 51)),
],
metadata=notebook_with_outputs.metadata,
)
write(notebook, "test.ipynb")
# try to commit it, should fail because
# 1. the py version hasn't been added
# 2. the first cell is '1+1' which is not black compliant
# 3. the second cell needs to be wrapped
tmp_repo.git.add("test.ipynb")
with pytest.raises(
HookExecutionError,
match="files were modified by this hook",
):
tmp_repo.index.commit("failing")
# Add the two files
tmp_repo.git.add("test.ipynb")
tmp_repo.git.add("test.py")
# now the commit will succeed
tmp_repo.index.commit("passing")
assert "test.ipynb" in tmp_repo.tree()
assert "test.py" in tmp_repo.tree()
# both the code and the markdown cells were reformatted
nb = read("test.ipynb")
assert nb.cells[0].source == "1 + 1"
print(nb.cells[1].source)
assert (nb.cells[1].source == """This is a complex markdown cell
# With a h1 header
## And a h2 header
| And | A | Table |
|-----|-----|-------|
| 0 | 1 | 2 |
!!!WARNING!!! This hook does not seem compatible with explicit paragraph
breaks (two spaces at the end of a line).
And a very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very very very very very
very very very very very very very very very very long line.""")