def test_doxygen_equation_markers(tmpdir):
cfg_file = tmpdir.join("jupytext.toml")
nb_file = tmpdir.join("notebook.ipynb")
md_file = tmpdir.join("notebook.md")
cfg_file.write(
"hide_notebook_metadata = true\ndoxygen_equation_markers = true")
nb = jupytext.reads(SAMPLE_MARKDOWN, "md")
del nb.metadata["jupytext"]
nbformat.write(nb, str(nb_file))
jupytext_cli([str(nb_file), "--to", "md"])
md = md_file.read()
# Doxygen equation markers
assert "$$" not in md
assert "\\f[" in md
assert "\\f]" in md
assert "\\f$" in md
assert "\\f$" in md
# Metadata hidden
assert md.startswith("<!--\n\n---\n")
jupytext_cli([str(md_file), "--to", "notebook"])
nb2 = nbformat.read(str(nb_file), as_version=4)
compare_notebooks(nb2, nb)
def test_custom_cell_magics(
tmpdir,
nb=new_notebook(cells=[
new_code_cell("%%sql -o tables -q\n SHOW TABLES"),
new_code_cell("""%%configure -f
{"executorMemory": "3072M", "executorCores": 4, "numExecutors":10}"""),
new_code_cell("%%local\na=1"),
]),
):
cfg_file = tmpdir.join("jupytext.toml")
nb_file = tmpdir.join("notebook.ipynb")
py_file = tmpdir.join("notebook.py")
cfg_file.write('custom_cell_magics = "configure,local"')
assert "configure" not in _JUPYTER_LANGUAGES_LOWER_AND_UPPER
assert "logs" not in _JUPYTER_LANGUAGES_LOWER_AND_UPPER
nbformat.write(nb, str(nb_file))
jupytext_cli([str(nb_file), "--to", "py"])
py = py_file.read()
for line in py.splitlines():
if line:
assert line.startswith("# "), line
jupytext_cli([str(py_file), "--to", "notebook"])
nb2 = nbformat.read(str(nb_file), as_version=4)
compare_notebooks(nb2, nb)
def test_reload_notebook_after_jupytext_cli(nb_file, tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_nbpy = str(tmpdir.join('notebook.py'))
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'py,ipynb'
cm.outdated_text_notebook_margin = 0
cm.root_dir = str(tmpdir)
# write the paired notebook
nb = jupytext.read(nb_file)
cm.save(model=dict(type='notebook', content=nb), path='notebook.py')
assert os.path.isfile(tmp_ipynb)
assert os.path.isfile(tmp_nbpy)
# run jupytext CLI
jupytext_cli([tmp_nbpy, '--to', 'ipynb', '--update'])
# test reload
nb1 = cm.get('notebook.py')['content']
nb2 = cm.get('notebook.ipynb')['content']
compare_notebooks(nb, nb1)
compare_notebooks(nb, nb2)
def test_jupytext_cli_bare(tmpdir):
tmp_py = str(tmpdir.join("test.py"))
tmp_ipynb = str(tmpdir.join("test.ipynb"))
write(new_notebook(cells=[new_code_cell("1 + 1")]), tmp_ipynb)
with pytest.warns(DeprecationWarning, match="nomarker"):
jupytext_cli([tmp_ipynb, "--to", "py:bare"])
assert os.path.isfile(tmp_py)
def test_jupytext_cli_bare(tmpdir):
tmp_py = str(tmpdir.join('test.py'))
tmp_ipynb = str(tmpdir.join('test.ipynb'))
write(new_notebook(cells=[new_code_cell('1 + 1')]), tmp_ipynb)
with pytest.warns(DeprecationWarning, match='nomarker'):
jupytext_cli([tmp_ipynb, '--to', 'py:bare'])
assert os.path.isfile(tmp_py)
def test_meaningfull_error_open_myst_missing(tmpdir):
md_file = tmpdir.join("notebook.md")
md_file.write(
"""---
jupytext:
text_representation:
extension: '.md'
format_name: myst
kernelspec:
display_name: Python 3
language: python
name: python3
---
1 + 1
"""
)
with pytest.raises(ImportError, match=PLEASE_INSTALL_MYST):
jupytext_cli([str(md_file), "--to", "ipynb"])
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
with pytest.raises(HTTPError, match=PLEASE_INSTALL_MYST):
cm.get("notebook.md")
def test_meaningfull_error_when_pandoc_is_missing(tmpdir):
nb_file = tmpdir.join("notebook.ipynb")
jupytext.write(new_notebook(), str(nb_file))
with pytest.raises(
PandocError,
match="The Pandoc Markdown format requires 'pandoc>=2.7.2'"):
jupytext_cli([str(nb_file), "--to", "md:pandoc"])
def test_execute_readme_and_update_index_html():
nb_path = os.path.dirname(os.path.abspath(__file__))
readme = os.path.join(nb_path, '..', 'README.md')
readme_ipynb = os.path.join(nb_path, '..', 'README.ipynb')
index = os.path.join(nb_path, '..', 'index.html')
jupytext_cli([readme, '--execute', '--to', 'ipynb'])
system('jupyter', 'nbconvert', '--to', 'html', readme_ipynb, '--output',
index)
def test_execute_readme_and_update_index_html():
nb_path = os.path.dirname(os.path.abspath(__file__))
readme = os.path.join(nb_path, "..", "README.md")
index = os.path.join(nb_path, "..", "index.html")
jupytext_cli([readme, "--execute", "--output", "README.ipynb"])
system("jupyter", "nbconvert", "--to", "html", "README.ipynb")
os.remove("README.ipynb")
if not os.path.isfile(index):
os.rename("README.html", index)
def test_myst_representation_same_cli_or_contents_manager(
tmpdir, cwd_tmpdir, notebook_with_outputs, language_info
):
"""This test gives some information on #759. As of Jupytext 1.11.1, in the MyST Markdown format,
the code cells have an ipython3 lexer when the notebook "language_info" metadata has "ipython3"
as the pygments_lexer. This information comes from the kernel and ATM it is not clear how the user
can choose to include it or not in the md file."""
nb = notebook_with_outputs
if language_info != "none":
nb["metadata"]["language_info"] = {
"codemirror_mode": {"name": "ipython", "version": 3},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3",
}
if language_info == "no_pygments_lexer":
del nb["metadata"]["language_info"]["pygments_lexer"]
# Writing the notebook with the Python API
text_api = jupytext.writes(nb, fmt="md:myst")
# How do code cells look like?
code_cells = set(
line for line in text_api.splitlines() if line.startswith("```{code-cell")
)
if language_info == "std":
assert code_cells == {"```{code-cell} ipython3"}
else:
assert code_cells == {"```{code-cell}"}
# We get the same file with the command line jupytext
tmpdir.mkdir("cli").join("notebook.ipynb").write(json.dumps(nb))
jupytext_cli(["--to", "md:myst", "cli/notebook.ipynb"])
text_cli = tmpdir.join("cli").join("notebook.md").read()
compare(text_cli, text_api)
# Or with the contents manager
cm = jupytext.TextFileContentsManager()
cm.formats = "ipynb,md:myst"
cm.root_dir = str(tmpdir.mkdir("contents_manager"))
cm.save(model=dict(content=nb, type="notebook"), path="notebook.ipynb")
text_cm = tmpdir.join("contents_manager").join("notebook.md").read()
compare(text_cm, text_api)
def test_root_level_metadata_as_raw_cell(
tmpdir,
root_level_metadata_as_raw_cell,
rmd="""---
author: R Markdown document author
title: R Markdown notebook title
---
```{r}
1 + 1
```
""",
):
nb_file = tmpdir.join("notebook.ipynb")
rmd_file = tmpdir.join("notebook.Rmd")
cfg_file = tmpdir.join("jupytext.toml")
cfg_file.write(
"root_level_metadata_as_raw_cell = {}".format(
"true" if root_level_metadata_as_raw_cell else "false"
)
)
rmd_file.write(rmd)
jupytext_cli([str(rmd_file), "--to", "ipynb"])
nb = nbformat.read(str(nb_file), as_version=4)
if root_level_metadata_as_raw_cell:
assert len(nb.cells) == 2
compare(
nb.cells[0],
new_raw_cell(
"""---
author: R Markdown document author
title: R Markdown notebook title
---"""
),
)
compare(nb.cells[1], new_code_cell("1 + 1"))
else:
assert len(nb.cells) == 1
compare(nb.cells[0], new_code_cell("1 + 1"))
assert nb.metadata["jupytext"]["root_level_metadata"] == {
"title": "R Markdown notebook title",
"author": "R Markdown document author",
}
# Writing back to Rmd should preserve the original document
jupytext_cli([str(nb_file), "--to", "Rmd"])
compare(rmd_file.read(), rmd)
def test_cli_expect_errors(tmp_ipynb):
with pytest.raises(ValueError):
jupytext([])
with pytest.raises(ValueError):
jupytext(['--sync'])
with pytest.raises(ValueError):
jupytext([tmp_ipynb, tmp_ipynb, '--paired-paths'])
with pytest.raises(ValueError):
jupytext(['--pre-commit', 'notebook.ipynb'])
with pytest.raises(ValueError):
jupytext(['notebook.ipynb', '--from', 'py:percent', '--to', 'md'])
with pytest.raises(SystemExit):
jupytext_cli([])
with pytest.raises((SystemExit, TypeError)): # SystemExit on Windows, TypeError on Linux
system('jupytext', ['notebook.ipynb', '--from', 'py:percent', '--to', 'md'])
def test_jupytext_commands_in_the_documentation_work(tmpdir):
# Read the documentation as a bash notebook
using_cli = os.path.join(doc_path, "using-cli.md")
assert os.path.isfile(using_cli)
using_cli_nb = jupytext.read(using_cli)
# Run the commands in tmpdir on a sample notebook
jupytext.write(new_notebook(cells=[new_code_cell("1+1")]),
str(tmpdir.join("notebook.ipynb")))
os.chdir(str(tmpdir))
cmd_tested = 0
for cell in using_cli_nb.cells:
if cell.cell_type != "code":
continue
if not cell.source.startswith("jupytext"):
continue
for cmd in cell.source.splitlines():
if not cmd.startswith("jupytext"):
continue
if "read ipynb from stdin" in cmd:
continue
if "pytest {}" in cmd:
continue
print("Testing: {}".format(cmd))
args = shlex.split(re.sub(r"#.*", "", cmd))[1:]
assert not jupytext_cli(args), cmd
cmd_tested += 1
assert cmd_tested >= 10
def test_jupytext_commands_in_the_documentation_work(tmpdir, nb_file):
# Read the documentation as a bash notebook
using_cli = os.path.join(doc_path, 'using-cli.md')
assert os.path.isfile(using_cli)
using_cli_nb = jupytext.read(using_cli)
# Run the commands in tmpdir on a sample notebook
shutil.copy(nb_file, str(tmpdir.join('notebook.ipynb')))
os.chdir(str(tmpdir))
cmd_tested = 0
for cell in using_cli_nb.cells:
if cell.cell_type != 'code':
continue
if not cell.source.startswith('jupytext'):
continue
for cmd in cell.source.splitlines():
if not cmd.startswith('jupytext'):
continue
if 'read ipynb from stdin' in cmd:
continue
if 'pytest {}' in cmd:
continue
print('Testing: {}'.format(cmd))
args = shlex.split(re.sub(r'#.*', '', cmd))[1:]
assert not jupytext_cli(args), cmd
cmd_tested += 1
assert cmd_tested >= 10
def test_default_config_has_priority_over_current_metadata(
tmpdir,
text="""# %% some_metadata_key=5
1 + 1
""",
):
py_file = tmpdir.join("notebook.py")
py_file.write(text)
cfg_file = tmpdir.join("jupytext.toml")
cfg_file.write("""cell_metadata_filter = "-some_metadata_key"
""")
jupytext_cli([str(py_file), "--to", "py"])
assert (py_file.read() == """# %%
1 + 1
""")
def test_metadata_filters_from_config(tmpdir):
cfg_file = tmpdir.join("jupytext.toml")
nb_file = tmpdir.join("notebook.ipynb")
md_file = tmpdir.join("notebook.md")
cfg_file.write("""default_notebook_metadata_filter = "-all"
default_cell_metadata_filter = "-all"
""")
nb = new_notebook(
cells=[new_markdown_cell("A markdown cell")],
metadata={
"kernelspec": {
"display_name": "Python [conda env:.conda-week1]",
"language": "python",
"name": "conda-env-.conda-week1-py",
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3",
},
"nbsphinx": {
"execute": "never"
},
},
)
nbformat.write(nb, str(nb_file))
jupytext_cli([str(nb_file), "--to", "md"])
md = md_file.read()
compare(md, "A markdown cell\n")
jupytext_cli([str(md_file), "--to", "notebook", "--update"])
nb2 = nbformat.read(str(nb_file), as_version=4)
del nb2.metadata["jupytext"]
compare_notebooks(nb2, nb)
def test_save_in_cm_matches_cli(tmpdir, cwd_tmpdir, fmt, input_fn):
# attributes of this format
long_fmt = long_form_one_format(fmt)
ext = long_fmt["extension"]
# format_name = long_fmt.get('format_name', 'markdown')
# the input file is md:myst
requires_myst()
# read sample input which is a md:myst notebook
from pathlib import Path
with (Path(__file__).parent / input_fn).open() as feed:
myst_input = feed.read()
nb = jupytext.reads(myst_input, "md:myst")
nb.metadata["jupytext"]["formats"] = fmt
# store notebook as ipynb in tmpdir
ipynb_path = str(tmpdir.join("notebook.ipynb"))
nbformat.write(nb, ipynb_path)
# invoke jupytext to produce version1
name1 = f"notebook1{ext}"
jupytext_cli(["--to", fmt, "-o", name1, "notebook.ipynb"])
# produce version2 with contents manager
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
name2 = f"notebook2{ext}"
cm.save(model=notebook_model(nb), path=name2)
text1 = tmpdir.join(name1).read()
text2 = tmpdir.join(name2).read()
compare(text1, text2)
# yet another tour
name3 = f"notebook3{ext}"
jupytext_cli(["--to", fmt, "-o", name3, name1])
text3 = tmpdir.join(name3).read()
compare(text1, text3)
def test_jupytext_commands_in_the_documentation_work(tmpdir):
# Read the documentation as a bash notebook
using_cli = os.path.join(doc_path, "using-cli.md")
assert os.path.isfile(using_cli)
using_cli_nb = jupytext.read(using_cli)
# Run the commands in tmpdir on a sample notebook
jupytext.write(new_notebook(cells=[new_code_cell("1+1")]),
str(tmpdir.join("notebook.ipynb")))
os.chdir(str(tmpdir))
cmd_tested = 0
for cell in using_cli_nb.cells:
if cell.cell_type != "code":
continue
if not cell.source.startswith("jupytext"):
continue
for cmd in cell.source.splitlines():
if not cmd.startswith("jupytext"):
continue
# Do not test commands that involve reading a notebook from stdin
if "read ipynb from stdin" in cmd:
continue
# We can't run pytest inside pytest
if "pytest {}" in cmd:
continue
# We need to remove the comments that may follow the jupytext command
if "#" in cmd:
left, comment = cmd.rsplit("#", 1)
if '"' not in comment:
cmd = left
print("Testing: {}".format(cmd))
args = shlex.split(cmd)[1:]
assert not jupytext_cli(args), cmd
cmd_tested += 1
assert cmd_tested >= 10
def test_meaningfull_error_write_myst_missing(tmpdir):
nb_file = tmpdir.join("notebook.ipynb")
jupytext.write(new_notebook(), str(nb_file))
with pytest.raises(ImportError, match=PLEASE_INSTALL_MYST):
jupytext_cli([str(nb_file), "--to", "md:myst"])
