def cells():
yield new_markdown_cell('<small>*Notebook auto-generated from '
'[``{0}``]({1})*</small>\n\n'
'# Altair Example: {2}\n\n'
'{3}\n\n'.format(full_filename, full_filepath,
title, description))
yield new_code_cell('# Uncomment and run these two lines to enable rendering in JupyterLab/nteract\n'
'# from altair import enable_mime_rendering\n'
'# enable_mime_rendering()')
yield new_markdown_cell('## Load Dataset\n'
'The data comes in the form of a Pandas '
'Dataframe:')
yield new_code_cell('from altair import load_dataset\n'
'data = load_dataset("{0}")\n'
'data.head()'.format(dataset))
yield new_markdown_cell('## Define Altair Specification')
yield new_code_cell(''.join(['from altair import * # Import the altair API\n\n',
'chart = {0}\n\n'.format(chart.to_altair(data='data')),
'chart.max_rows = len(data) # Altair 1.2.1 has a max_rows of 500 by default']))
yield new_markdown_cell('IPython rich display will invoke Vega-Lite:')
yield new_code_cell('chart')
yield new_markdown_cell('## Output Vega-Lite Specification')
yield new_markdown_cell('Generate JSON dict, leaving data out:')
yield new_code_cell('chart.to_dict(data=False)')
def test_notebook(name):
"""
Make a test notebook for the given name.
"""
nb = new_notebook()
nb.cells.append(new_code_cell("'code_' + '{}'".format(name)))
nb.cells.append(new_raw_cell("raw_{}".format(name)))
nb.cells.append(new_markdown_cell("markdown_{}".format(name)))
return nb
def cells():
yield new_markdown_cell('<small>*Notebook auto-generated from '
'[``{0}``]({1})*</small>\n\n'
'# Altair Example: {2}\n\n'
'{3}\n\n'.format(full_filename, full_filepath,
title, description))
yield new_markdown_cell('## Load Dataset\n'
'The data comes in the form of a Pandas '
'Dataframe:')
yield new_code_cell('from altair import load_dataset\n'
'data = load_dataset("{0}")\n'
'data.head()'.format(dataset))
yield new_markdown_cell('## Define Altair Specification')
yield new_code_cell('from altair import * # Import the altair API\n\n'
'chart = {0}'.format(chart.to_altair(data='data')))
yield new_markdown_cell('IPython rich display will invoke Vega-Lite:')
yield new_code_cell('chart')
yield new_markdown_cell('## Output Vega-Lite Specification')
yield new_markdown_cell('Generate JSON dict, leaving data out:')
yield new_code_cell('chart.to_dict(data=False)')
def create_code_cell(block):
"""Create a notebook code cell from a block."""
code_cell = nbbase.new_code_cell(source=block['content'])
attr = block['attributes']
if not attr.is_empty:
code_cell.metadata \
= nbbase.NotebookNode({'attributes': attr.to_dict()})
execution_count = attr.kvs.get('n')
if not execution_count:
code_cell.execution_count = None
else:
code_cell.execution_count = int(execution_count)
return code_cell
def writes(self, nb, metadata=None, **kwargs):
"""Return the text representation of the notebook"""
if self.fmt.get("format_name") == "pandoc":
metadata = insert_jupytext_info_and_filter_metadata(
metadata, self.ext, self.implementation)
cells = []
for cell in nb.cells:
cell_metadata = filter_metadata(
cell.metadata,
self.fmt.get("cell_metadata_filter"),
_IGNORE_CELL_METADATA,
)
if cell.cell_type == "code":
cells.append(
new_code_cell(source=cell.source,
metadata=cell_metadata))
else:
cells.append(
NotebookNode(
source=cell.source,
metadata=cell_metadata,
cell_type=cell.cell_type,
))
return notebook_to_md(
NotebookNode(
nbformat=nb.nbformat,
nbformat_minor=nb.nbformat_minor,
metadata=metadata,
cells=cells,
))
if self.fmt.get("format_name"
) == MYST_FORMAT_NAME or self.ext in myst_extensions(
no_md=True):
pygments_lexer = metadata.get("language_info",
{}).get("pygments_lexer", None)
metadata = insert_jupytext_info_and_filter_metadata(
metadata, self.ext, self.implementation)
cells = []
for cell in nb.cells:
cell_metadata = filter_metadata(
cell.metadata,
self.fmt.get("cell_metadata_filter"),
_IGNORE_CELL_METADATA,
)
if cell.cell_type == "code":
cells.append(
new_code_cell(source=cell.source,
metadata=cell_metadata))
else:
cells.append(
NotebookNode(
source=cell.source,
metadata=cell_metadata,
cell_type=cell.cell_type,
))
return notebook_to_myst(
NotebookNode(
nbformat=nb.nbformat,
nbformat_minor=nb.nbformat_minor,
metadata=metadata,
cells=cells,
),
default_lexer=pygments_lexer,
)
# Copy the notebook, in order to be sure we do not modify the original notebook
nb = NotebookNode(
nbformat=nb.nbformat,
nbformat_minor=nb.nbformat_minor,
metadata=deepcopy(metadata or nb.metadata),
cells=nb.cells,
)
metadata = nb.metadata
default_language = (default_language_from_metadata_and_ext(
metadata, self.implementation.extension, True) or "python")
self.update_fmt_with_notebook_options(nb.metadata)
if "use_runtools" not in self.fmt:
for cell in nb.cells:
if cell.metadata.get("hide_input", False) or cell.metadata.get(
"hide_output", False):
self.fmt["use_runtools"] = True
break
header = encoding_and_executable(nb, metadata, self.ext)
header_content, header_lines_to_next_cell = metadata_and_cell_to_header(
nb,
metadata,
self.implementation,
self.ext,
self.fmt.get("root_level_metadata_as_raw_cell", True),
)
header.extend(header_content)
cell_exporters = []
looking_for_first_markdown_cell = (
self.implementation.format_name
and self.implementation.format_name.startswith("sphinx"))
split_at_heading = self.fmt.get("split_at_heading", False)
for cell in nb.cells:
if looking_for_first_markdown_cell and cell.cell_type == "markdown":
cell.metadata.setdefault("cell_marker", '"""')
looking_for_first_markdown_cell = False
cell_exporters.append(
self.implementation.cell_exporter_class(
cell, default_language, self.fmt))
texts = [cell.cell_to_text() for cell in cell_exporters]
lines = []
# concatenate cells in reverse order to determine how many blank lines (pep8)
for i, cell in reversed(list(enumerate(cell_exporters))):
text = cell.remove_eoc_marker(texts[i], lines)
if (i == 0 and self.implementation.format_name
and self.implementation.format_name.startswith("sphinx")
and
(text in [["%matplotlib inline"], ["# %matplotlib inline"]])):
continue
lines_to_next_cell = cell.lines_to_next_cell
if lines_to_next_cell is None:
lines_to_next_cell = pep8_lines_between_cells(
text, lines, self.implementation.extension)
text.extend([""] * lines_to_next_cell)
# two blank lines between markdown cells in Rmd when those do not have explicit region markers
if self.ext in [".md", ".markdown", ".Rmd"] and not cell.is_code():
if (i + 1 < len(cell_exporters)
and not cell_exporters[i + 1].is_code()
and not texts[i][0].startswith("<!-- #")
and not texts[i + 1][0].startswith("<!-- #") and
(not split_at_heading or
not (texts[i + 1] and texts[i + 1][0].startswith("#")))):
text.append("")
# "" between two consecutive code cells in sphinx
if self.implementation.format_name.startswith(
"sphinx") and cell.is_code():
if i + 1 < len(cell_exporters) and cell_exporters[i +
1].is_code():
text.append('""')
lines = text + lines
if header_lines_to_next_cell is None:
header_lines_to_next_cell = pep8_lines_between_cells(
header_content, lines, self.implementation.extension)
header.extend([""] * header_lines_to_next_cell)
return "\n".join(header + lines)
def reads(self, s, **_):
"""Read a notebook represented as text"""
if self.fmt.get("format_name") == "pandoc":
return md_to_notebook(s)
if self.fmt.get("format_name") == MYST_FORMAT_NAME:
return myst_to_notebook(s)
lines = s.splitlines()
cells = []
metadata, jupyter_md, header_cell, pos = header_to_metadata_and_cell(
lines,
self.implementation.header_prefix,
self.implementation.extension,
self.fmt.get("root_level_metadata_as_raw_cell", True),
)
default_language = default_language_from_metadata_and_ext(
metadata, self.implementation.extension)
self.update_fmt_with_notebook_options(metadata)
if header_cell:
cells.append(header_cell)
lines = lines[pos:]
if (self.implementation.format_name
and self.implementation.format_name.startswith("sphinx")):
cells.append(new_code_cell(source="%matplotlib inline"))
cell_metadata_json = False
while lines:
reader = self.implementation.cell_reader_class(
self.fmt, default_language)
cell, pos = reader.read(lines)
cells.append(cell)
cell_metadata_json = cell_metadata_json or reader.cell_metadata_json
if pos <= 0:
raise Exception("Blocked at lines " +
"\n".join(lines[:6])) # pragma: no cover
lines = lines[pos:]
custom_cell_magics = self.fmt.get("custom_cell_magics", "").split(",")
set_main_and_cell_language(metadata, cells,
self.implementation.extension,
custom_cell_magics)
cell_metadata = set()
for cell in cells:
cell_metadata.update(cell.metadata.keys())
update_metadata_filters(metadata, jupyter_md, cell_metadata)
if cell_metadata_json:
metadata.setdefault("jupytext",
{}).setdefault("cell_metadata_json", True)
if (self.implementation.format_name
and self.implementation.format_name.startswith("sphinx")):
filtered_cells = []
for i, cell in enumerate(cells):
if (cell.source == "" and i > 0 and i + 1 < len(cells)
and cells[i - 1].cell_type != "markdown"
and cells[i + 1].cell_type != "markdown"):
continue
filtered_cells.append(cell)
cells = filtered_cells
return new_notebook(cells=cells, metadata=metadata)
def test_strict_raise_on_blank_line_removed():
ref = new_notebook(cells=[new_code_cell('1+1\n')])
test = new_notebook(cells=[new_code_cell('1+1')])
with pytest.raises(NotebookDifference):
compare_notebooks(test, ref, 'py:light', allow_expected_differences=False)
def test_raise_on_different_cell_metadata():
ref = new_notebook(cells=[new_code_cell('1+1')])
test = new_notebook(cells=[new_code_cell('1+1', metadata={'metakey': 'value'})])
with pytest.raises(NotebookDifference):
compare_notebooks(test, ref, 'py:light')
def test_pipe_into_autopep8():
nb_org = new_notebook(cells=[new_code_cell("1 +1")])
nb_dest = new_notebook(cells=[new_code_cell("1 + 1")])
nb_pipe = pipe_notebook(nb_org, "autopep8 -")
compare(nb_pipe, nb_dest)
# f.close()
# text = "".join(lines)
nb = nbformat.read(infilename, 4)
nb_new = nbformat.NotebookNode()
cells_new=[]
offset = 0
for cell in nb.cells:
if cell["cell_type"] == "code":
if cell["source"] == '%jsroot on':
offset = -1
else:
cells_new.append(new_code_cell(
source=cell.source,
metadata=cell.metadata,
outputs=[],
# print cell.execution_count
execution_count=cell.execution_count+offset))
# print cell.execution_count+offset
elif cell["cell_type"] == "markdown":
cells_new.append(new_markdown_cell(
source=cell.source,
metadata=cell.metadata))
else:
cells_new.append(new_raw_cell(
source=cell.source,
metadata=cell.metadata))
# print cell.cell_type
nb_new = new_notebook(cells=cells_new,
def test_read_magics(text="// :vars\n"):
nb = jupytext.reads(text, 'rs')
compare_notebooks(nb, new_notebook(cells=[new_code_cell(':vars')]))
compare(jupytext.writes(nb, 'rs'), text)
def test_read_simple_file(script="""# ---
# title: Simple file
# ---
# %% [markdown]
# This is a markdown cell
# %% [md]
# This is also a markdown cell
# %% [raw]
# This is a raw cell
# %%% sub-cell title
# This is a sub-cell
# %%%% sub-sub-cell title
# This is a sub-sub-cell
# %% And now a code cell
1 + 2 + 3 + 4
5
6
# %%magic # this is a commented magic, not a cell
7
""", ):
nb = jupytext.reads(script, "py:percent")
compare_notebooks(
new_notebook(cells=[
new_raw_cell("---\ntitle: Simple file\n---"),
new_markdown_cell("This is a markdown cell"),
new_markdown_cell("This is also a markdown cell",
metadata={"region_name": "md"}),
new_raw_cell("This is a raw cell"),
new_code_cell(
"# This is a sub-cell",
metadata={
"title": "sub-cell title",
"cell_depth": 1
},
),
new_code_cell(
"# This is a sub-sub-cell",
metadata={
"title": "sub-sub-cell title",
"cell_depth": 2
},
),
new_code_cell(
"""1 + 2 + 3 + 4
5
6
%%magic # this is a commented magic, not a cell
7""",
metadata={"title": "And now a code cell"},
),
]),
nb,
)
script2 = jupytext.writes(nb, "py:percent")
compare(script2, script)
import os
import sys
import unittest.mock as mock
import pytest
from nbformat import read
from nbformat.v4.nbbase import new_code_cell, new_markdown_cell, new_notebook
from tornado.web import HTTPError
import jupytext
from .utils import notebook_model
SAMPLE_NOTEBOOK = new_notebook(cells=[
new_markdown_cell("A Markdown cell"),
new_code_cell("# A code cell\n1 + 1")
])
def test_local_config_overrides_cm_config(tmpdir):
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.default_jupytext_formats = "ipynb,py"
nested = tmpdir.mkdir("nested")
with open(str(nested.join(".jupytext.yml")), "w") as fp:
fp.write("default_jupytext_formats: ''\n")
cm.save(notebook_model(SAMPLE_NOTEBOOK), "notebook.ipynb")
assert os.path.isfile(str(tmpdir.join("notebook.ipynb")))
assert os.path.isfile(str(tmpdir.join("notebook.py")))
def cells():
yield new_markdown_cell(''.join(comment_block))
yield new_code_cell(''.join(source_block))
def add_code_cell(cells, source):
"""Add a code cell containing `source` to the notebook."""
from nbformat.v4.nbbase import new_code_cell
n_code_cells = len([c for c in cells if c['cell_type'] == 'code'])
cells.append(new_code_cell(source=source,
execution_count=n_code_cells + 1))
def test_raise_on_different_cell_type(raise_on_first_difference):
ref = new_notebook(cells=[new_markdown_cell('Cell one'), new_code_cell('Cell two')])
test = new_notebook(cells=[new_markdown_cell('Cell one'), new_raw_cell('Cell two')])
with pytest.raises(NotebookDifference):
compare_notebooks(test, ref, 'md', raise_on_first_difference=raise_on_first_difference)
def notebook_1():
return new_notebook(
metadata=notebook_metadata(),
cells=[new_markdown_cell('First markdown cell'),
new_code_cell('1 + 1'),
new_markdown_cell('Second markdown cell')])
def test_339_ipynb(tmpdir, fmt):
tmp_ipynb = str(tmpdir.join('test.ipynb'))
nb = new_notebook(cells=[new_code_cell('cat = 42')])
nbformat.write(nb, tmp_ipynb)
assert jupytext([tmp_ipynb, '--to', fmt, '--test-strict']) == 0
def test_pipe_into_black():
nb_org = new_notebook(cells=[new_code_cell('1 +1')])
nb_dest = new_notebook(cells=[new_code_cell('1 + 1')])
nb_pipe = pipe_notebook(nb_org, 'black')
compare(nb_dest, nb_pipe)
def test_339_ipynb(tmpdir, fmt):
tmp_ipynb = str(tmpdir.join("test.ipynb"))
nb = new_notebook(cells=[new_code_cell("cat = 42")])
nbformat.write(nb, tmp_ipynb)
assert jupytext([tmp_ipynb, "--to", fmt, "--test-strict"]) == 0
def test_pipe_into_autopep8():
nb_org = new_notebook(cells=[new_code_cell('1 +1')])
nb_dest = new_notebook(cells=[new_code_cell('1 + 1')])
nb_pipe = pipe_notebook(nb_org, 'autopep8 -')
compare(nb_dest, nb_pipe)
def test_pipe_into_black():
nb_org = new_notebook(cells=[new_code_cell("1 +1")])
nb_dest = new_notebook(cells=[new_code_cell("1 + 1")])
nb_pipe = pipe_notebook(nb_org, "black")
compare(nb_pipe, nb_dest)
def add_code_cell(cells, source):
from nbformat.v4.nbbase import new_code_cell
n_code_cells = len([c for c in cells if c['cell_type']=='code'])
cells.append(new_code_cell(source=source,execution_count=n_code_cells+1))
def writes(self, nb, metadata=None, **kwargs):
"""Return the text representation of the notebook"""
if self.fmt.get('format_name') == 'pandoc':
metadata = insert_jupytext_info_and_filter_metadata(metadata, self.ext, self.implementation)
cells = []
for cell in nb.cells:
cell_metadata = filter_metadata(copy(cell.metadata),
self.fmt.get('cell_metadata_filter'),
_IGNORE_CELL_METADATA)
if cell.cell_type == 'code':
cells.append(new_code_cell(source=cell.source, metadata=cell_metadata))
else:
cells.append(NotebookNode(source=cell.source, metadata=cell_metadata, cell_type=cell.cell_type))
return notebook_to_md(NotebookNode(
nbformat=nb.nbformat,
nbformat_minor=nb.nbformat_minor,
metadata=metadata,
cells=cells))
# Copy the notebook, in order to be sure we do not modify the original notebook
nb = NotebookNode(
nbformat=nb.nbformat,
nbformat_minor=nb.nbformat_minor,
metadata=deepcopy(metadata or nb.metadata),
cells=nb.cells)
metadata = nb.metadata
default_language = default_language_from_metadata_and_ext(metadata, self.implementation.extension) or 'python'
self.update_fmt_with_notebook_options(nb.metadata)
if 'main_language' in metadata.get('jupytext', {}):
del metadata['jupytext']['main_language']
header = encoding_and_executable(nb, metadata, self.ext)
header_content, header_lines_to_next_cell = metadata_and_cell_to_header(nb, metadata,
self.implementation, self.ext)
header.extend(header_content)
cell_exporters = []
looking_for_first_markdown_cell = (self.implementation.format_name and
self.implementation.format_name.startswith('sphinx'))
split_at_heading = self.fmt.get('split_at_heading', False)
for cell in nb.cells:
if looking_for_first_markdown_cell and cell.cell_type == 'markdown':
cell.metadata.setdefault('cell_marker', '"""')
looking_for_first_markdown_cell = False
cell_exporters.append(self.implementation.cell_exporter_class(cell, default_language, self.fmt))
texts = [cell.cell_to_text() for cell in cell_exporters]
lines = []
# concatenate cells in reverse order to determine how many blank lines (pep8)
for i, cell in reversed(list(enumerate(cell_exporters))):
text = cell.remove_eoc_marker(texts[i], lines)
if i == 0 and self.implementation.format_name and \
self.implementation.format_name.startswith('sphinx') and \
(text in [['%matplotlib inline'], ['# %matplotlib inline']]):
continue
lines_to_next_cell = cell.lines_to_next_cell
if lines_to_next_cell is None:
lines_to_next_cell = pep8_lines_between_cells(text, lines, self.implementation.extension)
text.extend([''] * lines_to_next_cell)
# two blank lines between markdown cells in Rmd when those do not have explicit region markers
if self.ext in ['.Rmd', '.md'] and not cell.is_code():
if (i + 1 < len(cell_exporters) and not cell_exporters[i + 1].is_code() and
not texts[i][0].startswith('<!-- #region') and
not texts[i + 1][0].startswith('<!-- #region') and
(not split_at_heading or not (texts[i + 1] and texts[i + 1][0].startswith('#')))):
text.append('')
# "" between two consecutive code cells in sphinx
if self.implementation.format_name.startswith('sphinx') and cell.is_code():
if i + 1 < len(cell_exporters) and cell_exporters[i + 1].is_code():
text.append('""')
if i + 1 < len(cell_exporters):
lines = cell_exporters[i + 1].simplify_soc_marker(lines, text)
lines = text + lines
if header_lines_to_next_cell is None:
header_lines_to_next_cell = pep8_lines_between_cells(header_content, lines, self.implementation.extension)
header.extend([''] * header_lines_to_next_cell)
if cell_exporters:
lines = cell_exporters[0].simplify_soc_marker(lines, header)
return '\n'.join(header + lines)
def add_code_cell(cells, source):
"""Add a code cell containing `source` to the notebook."""
from nbformat.v4.nbbase import new_code_cell
n_code_cells = len([c for c in cells if c['cell_type'] == 'code'])
cells.append(new_code_cell(source=source, execution_count=n_code_cells+1))
import os
try:
import unittest.mock as mock
except ImportError:
import mock
from nbformat.v4.nbbase import new_notebook, new_markdown_cell, new_code_cell
from nbformat import read
import jupytext
SAMPLE_NOTEBOOK = new_notebook(
cells=[new_markdown_cell("A Markdown cell"), new_code_cell("# A code cell\n1 + 1")]
)
def test_local_config_overrides_cm_config(tmpdir):
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.default_jupytext_formats = "ipynb,py"
nested = tmpdir.mkdir("nested")
with open(str(nested.join(".jupytext.yml")), "w") as fp:
fp.write("default_jupytext_formats: ''\n")
cm.save(dict(type="notebook", content=SAMPLE_NOTEBOOK), "notebook.ipynb")
assert os.path.isfile(str(tmpdir.join("notebook.ipynb")))
assert os.path.isfile(str(tmpdir.join("notebook.py")))
cm.save(dict(type="notebook", content=SAMPLE_NOTEBOOK), "nested/notebook.ipynb")
assert os.path.isfile(str(nested.join("notebook.ipynb")))
assert not os.path.isfile(str(nested.join("notebook.py")))
def test_does_not_raise_on_blank_line_removed():
ref = new_notebook(cells=[new_code_cell('1+1\n ')])
test = new_notebook(cells=[new_code_cell('1+1')])
compare_notebooks(test, ref, 'py:light')
def test_mutiple_cells_differ():
nb1 = new_notebook(cells=[new_code_cell(''), new_code_cell('2')])
nb2 = new_notebook(cells=[new_code_cell('1+1'), new_code_cell('2\n2')])
with pytest.raises(NotebookDifference) as exception_info:
compare_notebooks(nb2, nb1, raise_on_first_difference=False)
assert 'Cells 1,2 differ' in exception_info.value.args[0]
def test_read_mostly_py_markdown_file(
markdown="""---
title: Simple file
---
```python
import numpy as np
x = np.arange(0, 2*math.pi, eps)
```
```python
x = np.arange(0,1,eps)
y = np.abs(x)-.5
```
This is
a Markdown cell
```
# followed by a code cell with no language info
```
```
# another code cell
# with two blank lines
```
And the same markdown cell continues
<!-- #raw -->
this is a raw cell
<!-- #endraw -->
```R
ls()
```
```R
cat(stringi::stri_rand_lipsum(3), sep='\n\n')
```
""",
):
nb = jupytext.reads(markdown, "md")
assert nb.metadata["jupytext"]["main_language"] == "python"
compare_cells(
nb.cells,
[
new_raw_cell("---\ntitle: Simple file\n---"),
new_code_cell("import numpy as np\n" "x = np.arange(0, 2*math.pi, eps)"),
new_code_cell("x = np.arange(0,1,eps)\ny = np.abs(x)-.5"),
new_markdown_cell(
"""This is
a Markdown cell
```
# followed by a code cell with no language info
```
```
# another code cell
# with two blank lines
```
And the same markdown cell continues"""
),
new_raw_cell("this is a raw cell"),
new_code_cell("%%R\nls()"),
new_code_cell("%%R\ncat(stringi::" "stri_rand_lipsum(3), sep='\n\n')"),
],
compare_ids=False,
)
markdown2 = jupytext.writes(nb, "md")
compare(markdown2, markdown)
def block_code(self, code, lang=None):
return nbbase.new_code_cell(source=code)
def add_code_cell(cells, source):
from nbformat.v4.nbbase import new_code_cell
n_code_cells = len([c for c in cells if c['cell_type'] == 'code'])
cells.append(new_code_cell(source=source,
execution_count=n_code_cells + 1))
