def test_notebook(name):
"""
Make a test notebook for the given name.
"""
nb = new_notebook()
nb.cells.append(new_code_cell("'code_' + '{}'".format(name)))
nb.cells.append(new_raw_cell("raw_{}".format(name)))
nb.cells.append(new_markdown_cell("markdown_{}".format(name)))
return nb
def write_notebook(cells, outputfile, execute=True, kernel='python3'):
kernelspec = get_kernelspec(kernel)
notebook = new_notebook(cells=cells,
metadata={'language': 'python',
'kernelspec': kernelspec})
if execute:
ep = ExecutePreprocessor(timeout=600, kernelname='python3')
ep.preprocess(notebook,
{'metadata': {'path': os.path.dirname(outputfile)}})
nbformat.write(notebook, outputfile)
def write_nb(root, nb_name, cells):
"""Write a jupyter notebook to disk.
Takes a given a root directory, a notebook name, and a list of cells.
"""
nb = new_notebook(cells=cells,
metadata={
'language': 'python',
})
nb_path = os.path.join(root, '%s.ipynb' % nb_name)
with codecs.open(nb_path, encoding='utf-8', mode='w') as nb_file:
nbformat.write(nb, nb_file, NB_VERSION)
print("Created Jupyter notebook at:\n%s" % nb_path)
def test_vscode_pycharm_folding_markers(tmpdir):
tmp_ipynb = str(tmpdir.join('nb.ipynb'))
tmp_py = str(tmpdir.join('nb.py'))
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
# Default VScode/PyCharm folding markers
cm.default_cell_markers = 'region,endregion'
cm.default_jupytext_formats = 'ipynb,py'
nb = new_notebook(cells=[
new_code_cell("""# {{{
'''Sample cell with region markers'''
'''End of the cell'''
# }}}"""),
new_code_cell('a = 1\n\n\nb = 1')
])
cm.save(model=dict(content=nb, type='notebook'), path='nb.ipynb')
assert os.path.isfile(tmp_ipynb)
assert os.path.isfile(tmp_py)
nb2 = cm.get('nb.ipynb')['content']
compare_notebooks(nb2, nb)
nb3 = read(tmp_py)
assert nb3.metadata['jupytext']['cell_markers'] == 'region,endregion'
with open(tmp_py) as fp:
text = fp.read()
# Remove YAML header
text = re.sub(re.compile(r'# ---.*# ---\n\n', re.DOTALL), '', text)
compare(
"""# {{{
'''Sample cell with region markers'''
'''End of the cell'''
# }}}
# region
a = 1
b = 1
# endregion
""", text)
def test_markdown_and_r_extensions(tmpdir):
tmp_r = str(tmpdir.join('script.r'))
tmp_markdown = str(tmpdir.join('notebook.markdown'))
nb = new_notebook()
write(nb, tmp_r)
write(nb, tmp_markdown)
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
model = cm.get('script.r')
assert model['type'] == 'notebook'
model = cm.get('notebook.markdown')
assert model['type'] == 'notebook'
def test_active_tag(text='''# + tags=["active-py"]
interpreter = 'python'
# + tags=["active-ipynb"]
# interpreter = 'ipython'
''',
ref=new_notebook(cells=[
new_raw_cell("interpreter = 'python'",
metadata={'tags': ['active-py']}),
new_code_cell("interpreter = 'ipython'",
metadata={'tags': ['active-ipynb']})
])):
nb = jupytext.reads(text, 'py')
compare_notebooks(nb, ref)
py = jupytext.writes(nb, 'py')
compare(py, text)
def test_multiline_comments_format_option():
text = '''# %% [markdown]
"""
a
long
cell
"""
'''
nb = new_notebook(
cells=[new_markdown_cell("a\nlong\ncell")],
metadata={
"jupytext": {"cell_markers": '"""', "notebook_metadata_filter": "-all"}
},
)
py = jupytext.writes(nb, "py:percent")
compare(py, text)
def write_notebook(cells, outputfile, execute=True, kernel='python3'):
kernelspec = get_kernelspec(kernel)
notebook = new_notebook(cells=cells,
metadata={
'language': 'python',
'kernelspec': kernelspec
})
if execute:
ep = ExecutePreprocessor(timeout=600, kernelname='python3')
ep.preprocess(notebook,
{'metadata': {
'path': os.path.dirname(outputfile)
}})
nbformat.write(notebook, outputfile)
def test_hide_code_tag(
no_jupytext_version_number,
nb=new_notebook(
metadata={"jupytext": {
"main_language": "matlab"
}},
cells=[new_code_cell("1 + 1", metadata={"tags": ["hide_code"]})],
),
text="""% + tags=["hide_code"]
1 + 1
""",
):
text2 = jupytext.writes(nb, "m")
compare(text2, text)
nb2 = jupytext.reads(text, "m")
compare_notebooks(nb2, nb)
def test_docstring_with_quadruple_quote(nb=new_notebook(cells=[
new_code_cell('''def fun_1(df):
""""
docstring starting with 4 double quotes and ending with 3
"""
return df'''),
new_code_cell('''def fun_2(df):
"""
docstring
"""
return df'''),
])):
"""Reproduces https://github.com/mwouts/jupytext/issues/460"""
py = jupytext.writes(nb, "py:percent")
nb2 = jupytext.reads(py, "py")
compare_notebooks(nb2, nb)
def test_set_option_split_at_heading(tmpdir):
tmp_rmd = tmpdir.join("notebook.Rmd")
tmp_rmd.write("""A paragraph
# H1 Header
""")
jupytext([str(tmp_rmd), "--opt", "split_at_heading=true"])
assert "split_at_heading: true" in tmp_rmd.read()
nb = read(str(tmp_rmd))
nb_expected = new_notebook(cells=[
new_markdown_cell("A paragraph"),
new_markdown_cell("# H1 Header")
])
compare_notebooks(nb, nb_expected)
def test_metadata_and_cell_to_header():
nb = new_notebook(metadata={'jupytext': {
'mainlanguage': 'python'
}},
cells=[
new_raw_cell(source="---\ntitle: Sample header\n---",
metadata={'noskipline': True})
])
header = metadata_and_cell_to_header(nb, get_format('.md'), '.md')
assert '\n'.join(header) == """---
title: Sample header
jupyter:
jupytext:
mainlanguage: python
---"""
assert nb.cells == []
def test_raise_on_wrong_format(tmpdir):
tmp_ipynb = str(tmpdir.join("notebook.ipynb"))
cm = TextFileContentsManager()
cm.root_dir = str(tmpdir)
with pytest.raises(HTTPError):
cm.save(
path=tmp_ipynb,
model=dict(
type="notebook",
content=new_notebook(
nbformat=4, metadata=dict(jupytext_formats=[".doc"])
),
),
)
def test_pair_in_tree_and_parent(tmpdir):
nb_file = (tmpdir.mkdir("notebooks").mkdir("subfolder").mkdir("a").mkdir(
"b").join("example.ipynb"))
py_file = tmpdir.mkdir("scripts").mkdir("subfolder").mkdir("c").join(
"example.py")
write(new_notebook(cells=[new_markdown_cell("A markdown cell")]),
str(nb_file))
jupytext([
"--set-formats", "notebooks//a/b//ipynb,scripts//c//py:percent",
str(nb_file)
])
assert py_file.exists()
assert "A markdown cell" in py_file.read()
def test_multiple_empty_cells():
nb = new_notebook(
cells=[new_code_cell(), new_code_cell(), new_code_cell()],
metadata={"jupytext": {"notebook_metadata_filter": "-all"}},
)
text = jupytext.writes(nb, "py:percent")
expected = """# %%
# %%
# %%
"""
compare(text, expected)
nb2 = jupytext.reads(text, "py:percent")
nb2.metadata = nb.metadata
compare_notebooks(nb2, nb)
def test_no_files_created_on_no_format(tmpdir):
tmp_ipynb = "notebook.ipynb"
tmp_rmd = "notebook.Rmd"
tmp_py = "notebook.py"
cm = TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.default_jupytext_formats = ""
cm.save(
model=notebook_model(new_notebook(nbformat=4, metadata=dict())),
path=tmp_ipynb,
)
assert not os.path.isfile(str(tmpdir.join(tmp_py)))
assert not os.path.isfile(str(tmpdir.join(tmp_rmd)))
def test_round_trip_python_with_js_cell_no_cell_metadata():
notebook = new_notebook(cells=[
new_code_cell('''import notebook.nbextensions
notebook.nbextensions.install_nbextension('jupytext.js', user=True)'''),
new_code_cell('''%%javascript
Jupyter.utils.load_extensions('jupytext')''')
],
metadata={
'jupytext': {
'notebook_metadata_filter': '-all',
'cell_metadata_filter': '-all'
}
})
text = jupytext.writes(notebook, 'py')
notebook2 = jupytext.reads(text, 'py')
compare_notebooks(notebook, notebook2)
def test_save_ipynb_with_jupytext_has_final_newline(tmpdir):
nb = new_notebook()
file_jupytext = str(tmpdir.join('jupytext.ipynb'))
file_nbformat = str(tmpdir.join('nbformat.ipynb'))
jupytext.write(nb, file_jupytext)
with open(file_nbformat, 'w') as fp:
nbformat.write(nb, fp)
with open(file_jupytext) as fp:
text_jupytext = fp.read()
with open(file_nbformat) as fp:
text_nbformat = fp.read()
compare(text_jupytext, text_nbformat)
def test_force_comment_using_contents_manager(tmpdir):
tmp_py = 'notebook.py'
cm = jupytext.TextFileContentsManager()
cm.preferred_jupytext_formats_save = 'py:percent'
cm.root_dir = str(tmpdir)
nb = new_notebook(cells=[new_code_cell('%pylab inline')])
cm.save(model=dict(type='notebook', content=nb), path=tmp_py)
with open(str(tmpdir.join(tmp_py))) as stream:
assert '# %pylab inline' in stream.read().splitlines()
cm.comment_magics = False
cm.save(model=dict(type='notebook', content=nb), path=tmp_py)
with open(str(tmpdir.join(tmp_py))) as stream:
assert '%pylab inline' in stream.read().splitlines()
def test_split_at_heading_in_metadata(
markdown="""---
jupyter:
jupytext:
split_at_heading: true
---
A paragraph
# H1 Header
""",
nb_expected=new_notebook(
cells=[new_markdown_cell("A paragraph"), new_markdown_cell("# H1 Header")]
),
):
nb = jupytext.reads(markdown, ".md")
compare_notebooks(nb, nb_expected)
def test_inactive_cell(
text="""```python active="md"
# This becomes a raw cell in Jupyter
```
""",
expected=new_notebook(
cells=[
new_raw_cell(
"# This becomes a raw cell in Jupyter", metadata={"active": "md"}
)
]
),
):
nb = jupytext.reads(text, "md")
compare_notebooks(nb, expected)
text2 = jupytext.writes(nb, "md")
compare(text2, text)
def test_rename_inconsistent_path(tmpdir):
org_file = str(tmpdir.join('notebook_suffix.ipynb'))
new_file = str(tmpdir.join('new.ipynb'))
jupytext.writef(
new_notebook(metadata={'jupytext': {
'formats': '_suffix.ipynb'
}}), org_file)
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
# Read notebook, and learn about its format
cm.get('notebook_suffix.ipynb')
with pytest.raises(HTTPError):
cm.rename_file('notebook_suffix.ipynb', 'new.ipynb')
assert not os.path.isfile(new_file)
assert os.path.isfile(org_file)
def to_notebook(self, s, **kwargs):
"""Convert the markdown string s to an IPython notebook.
Returns a notebook.
"""
all_blocks = self.parse_blocks(s)
if self.pre_code_block['content']:
# TODO: if first block is markdown, place after?
all_blocks.insert(0, self.pre_code_block)
blocks = [self.process_code_block(block) for block in all_blocks]
cells = self.create_cells(blocks)
nb = nbbase.new_notebook(cells=cells)
return nb
def reads(self, s, **_):
"""Read a notebook represented as text"""
if self.fmt.get('format_name') == 'pandoc':
return md_to_notebook(s)
lines = s.splitlines()
cells = []
metadata, jupyter_md, header_cell, pos = header_to_metadata_and_cell(lines,
self.implementation.header_prefix,
self.implementation.extension)
default_language = default_language_from_metadata_and_ext(metadata, self.implementation.extension)
self.update_fmt_with_notebook_options(metadata)
if header_cell:
cells.append(header_cell)
lines = lines[pos:]
if self.implementation.format_name and self.implementation.format_name.startswith('sphinx'):
cells.append(new_code_cell(source='%matplotlib inline'))
cell_metadata = set()
while lines:
reader = self.implementation.cell_reader_class(self.fmt, default_language)
cell, pos = reader.read(lines)
cells.append(cell)
cell_metadata.update(cell.metadata.keys())
if pos <= 0:
raise Exception('Blocked at lines ' + '\n'.join(lines[:6])) # pragma: no cover
lines = lines[pos:]
update_metadata_filters(metadata, jupyter_md, cell_metadata)
set_main_and_cell_language(metadata, cells, self.implementation.extension)
if self.implementation.format_name and self.implementation.format_name.startswith('sphinx'):
filtered_cells = []
for i, cell in enumerate(cells):
if cell.source == '' and i > 0 and i + 1 < len(cells) \
and cells[i - 1].cell_type != 'markdown' and cells[i + 1].cell_type != 'markdown':
continue
filtered_cells.append(cell)
cells = filtered_cells
return new_notebook(cells=cells, metadata=metadata)
def to_notebook(self, s, **kwargs):
"""Convert the markdown string s to an IPython notebook.
Returns a notebook.
"""
all_blocks = self.parse_blocks(s)
if self.pre_code_block['content']:
# TODO: if first block is markdown, place after?
all_blocks.insert(0, self.pre_code_block)
blocks = [self.process_code_block(block) for block in all_blocks]
cells = self.create_cells(blocks)
nb = nbbase.new_notebook(cells=cells)
return nb
def test_skip_execution(tmpdir, cwd_tmpdir, tmp_repo, python_notebook, capsys):
write(
new_notebook(cells=[new_code_cell("1 + 1")], metadata=python_notebook.metadata),
"test.ipynb",
)
tmp_repo.index.add("test.ipynb")
jupytext(["--execute", "--pre-commit-mode", "test.ipynb"])
captured = capsys.readouterr()
assert "Executing notebook" in captured.out
nb = read("test.ipynb")
assert nb.cells[0].execution_count == 1
jupytext(["--execute", "--pre-commit-mode", "test.ipynb"])
captured = capsys.readouterr()
assert "skipped" in captured.out
def test_single_triple_quote_works(
no_jupytext_version_number,
text='''# ---
# jupyter:
# jupytext:
# cell_markers: '"""'
# formats: ipynb,py:percent
# text_representation:
# extension: .py
# format_name: percent
# ---
# %%
print("hello")
''',
notebook=new_notebook(cells=[new_code_cell('print("hello")')])):
compare_notebooks(jupytext.reads(text, 'py'), notebook)
def test_inactive_cell_using_tag(
text="""```python tags=["active-md"]
# This becomes a raw cell in Jupyter
```
""",
expected=new_notebook(
cells=[
new_raw_cell(
"# This becomes a raw cell in Jupyter", metadata={"tags": ["active-md"]}
)
]
),
):
nb = jupytext.reads(text, "md")
compare_notebooks(nb, expected)
text2 = jupytext.writes(nb, "md")
compare(text2, text)
def test_multiline_comments_format_option():
text = '''# %% [markdown]
"""
a
long
cell
"""
'''
nb = new_notebook(cells=[new_markdown_cell("a\nlong\ncell")],
metadata={
'jupytext': {
'cell_markers': '"""',
'notebook_metadata_filter': '-all'
}
})
py = jupytext.writes(nb, 'py:percent')
compare(py, text)
def test_no_files_created_on_no_format(tmpdir):
tmp_ipynb = 'notebook.ipynb'
tmp_rmd = 'notebook.Rmd'
tmp_py = 'notebook.py'
cm = TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.default_jupytext_formats = ''
cm.save(
model=dict(type='notebook',
content=new_notebook(nbformat=4,
metadata=dict())),
path=tmp_ipynb)
assert not os.path.isfile(str(tmpdir.join(tmp_py)))
assert not os.path.isfile(str(tmpdir.join(tmp_rmd)))
def test_force_comment_using_contents_manager(tmpdir):
tmp_py = "notebook.py"
cm = jupytext.TextFileContentsManager()
cm.preferred_jupytext_formats_save = "py:percent"
cm.root_dir = str(tmpdir)
nb = new_notebook(cells=[new_code_cell("%pylab inline")])
cm.save(model=notebook_model(nb), path=tmp_py)
with open(str(tmpdir.join(tmp_py))) as stream:
assert "# %pylab inline" in stream.read().splitlines()
cm.comment_magics = False
cm.save(model=notebook_model(nb), path=tmp_py)
with open(str(tmpdir.join(tmp_py))) as stream:
assert "%pylab inline" in stream.read().splitlines()
def write_nb(root: str, nb_name: str, cells: list) -> None:
"""Write a jupyter notebook to disk.
Takes a given a root directory, a notebook name, and a list of cells.
Args:
root (str): The root node (section) of the notebook.
nb_name (str): The name of the notebook file.
cells (list): The list of the cells of the notebook.
"""
nb = new_notebook(cells=cells, metadata={
'language': 'python',
})
nb_path = root / ('%s.ipynb' % nb_name)
with codecs.open(nb_path, encoding='utf-8', mode='w') as nb_file:
nbformat.write(nb, nb_file, NB_VERSION)
print("Created Jupyter notebook at:\n%s" % nb_path)
def test_questions_in_unmarked_cells_are_not_uncommented_297(
text="""# This cell has no explicit marker
# question?
1 + 2
""",
nb=new_notebook(cells=[
new_code_cell(
"# This cell has no explicit marker\n# question?\n1 + 2",
metadata={"comment_questions": False},
)
]),
):
nb2 = jupytext.reads(text, "py")
compare_notebooks(nb2, nb)
text2 = jupytext.writes(nb2, "py")
compare(text2, text)
def test_metadata_filters_from_config(tmpdir):
cfg_file = tmpdir.join("jupytext.toml")
nb_file = tmpdir.join("notebook.ipynb")
md_file = tmpdir.join("notebook.md")
cfg_file.write("""default_notebook_metadata_filter = "-all"
default_cell_metadata_filter = "-all"
""")
nb = new_notebook(
cells=[new_markdown_cell("A markdown cell")],
metadata={
"kernelspec": {
"display_name": "Python [conda env:.conda-week1]",
"language": "python",
"name": "conda-env-.conda-week1-py",
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3",
},
"nbsphinx": {
"execute": "never"
},
},
)
nbformat.write(nb, str(nb_file))
jupytext_cli([str(nb_file), "--to", "md"])
md = md_file.read()
compare(md, "A markdown cell\n")
jupytext_cli([str(md_file), "--to", "notebook", "--update"])
nb2 = nbformat.read(str(nb_file), as_version=4)
del nb2.metadata["jupytext"]
compare_notebooks(nb2, nb)
def create_doc_notebook(filename):
kernelspec = {'display_name': 'Python 3',
'language': 'python',
'name': 'python3'}
is_toplevel = lambda obj: 'Chart' in obj['name']
content = sorted(altair_nbdoc(altair), key=itemgetter('name'))
cells = [new_markdown_cell(source=(HEADER))]
cells.append(new_markdown_cell(source='## Top-Level Objects'))
cells.extend([new_markdown_cell(source=TEMPLATE.render(obj=obj))
for obj in filter(is_toplevel, content)])
cells.append(new_markdown_cell(source='## Other Objects'))
cells.extend([new_markdown_cell(source=TEMPLATE.render(obj=obj))
for obj in filterfalse(is_toplevel, content)])
notebook = new_notebook(cells=cells,
metadata={'language': 'python',
'kernelspec': kernelspec})
nbformat.write(notebook, filename)
execution_count=cell.execution_count+offset))
# print cell.execution_count+offset
elif cell["cell_type"] == "markdown":
cells_new.append(new_markdown_cell(
source=cell.source,
metadata=cell.metadata))
else:
cells_new.append(new_raw_cell(
source=cell.source,
metadata=cell.metadata))
# print cell.cell_type
nb_new = new_notebook(cells=cells_new,
metadata=nb.metadata,
nbformat=nb.nbformat,
nbformat_minor=nb.nbformat_minor
)
with codecs.open(outfilename, encoding='utf-8', mode='w') as out:
nbformat.write(nb_new, out)
# nbformat.write(nb, fp, version=nbformat.NO_CONVERT, **kwargs)
#################################################################
# true=True
# false=False
# nbj = eval(text)
# cells = nbj["cells"]