def test_combine_lower_version_raises(tmpdir):
tmp_ipynb = 'notebook.ipynb'
tmp_nbpy = 'notebook.py'
with open(str(tmpdir.join(tmp_nbpy)), 'w') as fp:
fp.write("""# ---
# jupyter:
# jupytext_formats: ipynb,py
# jupytext_format_version: '0.0'
# ---
# New cell
""")
nb = new_notebook(metadata={'jupytext_formats': 'ipynb,py'})
jupytext.writef(nb, str(tmpdir.join(tmp_ipynb)))
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'ipynb,py'
cm.root_dir = str(tmpdir)
with pytest.raises(HTTPError):
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER',
True):
cm.get(tmp_ipynb)
def test_combine_lower_version_raises(tmpdir):
tmp_ipynb = 'notebook.ipynb'
tmp_nbpy = 'notebook.py'
with open(str(tmpdir.join(tmp_nbpy)), 'w') as fp:
fp.write("""# ---
# jupyter:
# jupytext_formats: ipynb,py
# jupytext_format_version: '0.0'
# ---
# New cell
""")
nb = new_notebook(metadata={'jupytext_formats': 'ipynb,py'})
jupytext.writef(nb, str(tmpdir.join(tmp_ipynb)))
cm = TextFileContentsManager()
cm.default_jupytext_formats = 'ipynb,py'
cm.root_dir = str(tmpdir)
with pytest.raises(HTTPError):
with mock.patch('jupytext.file_format_version.FILE_FORMAT_VERSION',
{'.py': '1.0'}):
with mock.patch(
'jupytext.file_format_version.MIN_FILE_FORMAT_VERSION',
{'.py': '1.0'}):
cm.get(tmp_ipynb)
def create_notebook(did, name):
snippet = snippet_header() + \
snippet_object(did)
# Get the IPython script root dir
root_dir = pathlib.Path.cwd().joinpath("")
kernelspec = """ {"kernelspec" : {
"display_name": "Manta Ray",
"language": "python",
"name": "python3"
}}
""".replace("\n", "")
kernel_spec_dict = json.loads(kernelspec)
# The output path
jupyter_file_name = name + '.ipynb'
out_path = root_dir / jupyter_file_name
# Parse the script to Jupyter format
parsed = jupytext.reads(snippet, fmt='.py', format_name='percent')
parsed['metadata'].update(kernel_spec_dict)
# Delete the file if it exists
if out_path.exists():
out_path.unlink()
# Write the result
jupytext.writef(parsed, out_path)
return snippet
def test_combine_same_version_ok(tmpdir):
tmp_ipynb = 'notebook.ipynb'
tmp_nbpy = 'notebook.py'
with open(str(tmpdir.join(tmp_nbpy)), 'w') as fp:
fp.write("""# ---
# jupyter:
# jupytext_formats: ipynb,py
# jupytext_format_version: '1.2'
# ---
# New cell
""")
nb = new_notebook(metadata={'jupytext_formats': 'ipynb,py'})
jupytext.writef(nb, str(tmpdir.join(tmp_ipynb)))
cm = jupytext.TextFileContentsManager()
cm.default_jupytext_formats = 'ipynb,py'
cm.root_dir = str(tmpdir)
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True):
nb = cm.get(tmp_ipynb)
cells = nb['content']['cells']
assert len(cells) == 1
assert cells[0].cell_type == 'markdown'
assert cells[0].source == 'New cell'
def test_cli_sync_file_with_suffix(tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_pct_py = str(tmpdir.join('notebook.pct.py'))
tmp_lgt_py = str(tmpdir.join('notebook.lgt.py'))
tmp_rmd = str(tmpdir.join('notebook.Rmd'))
nb = new_notebook(cells=[new_code_cell(source='1+1')],
metadata={
'jupytext': {
'formats':
'ipynb,.pct.py:percent,.lgt.py:light,Rmd'
}
})
writef(nb, tmp_pct_py, '.pct.py:percent')
jupytext(['--sync', tmp_pct_py])
assert os.path.isfile(tmp_lgt_py)
assert os.path.isfile(tmp_rmd)
assert os.path.isfile(tmp_ipynb)
jupytext(['--sync', tmp_lgt_py])
jupytext(['--sync', tmp_ipynb])
assert open(tmp_lgt_py).read().splitlines()[-2:] == ['', '1+1']
assert open(tmp_pct_py).read().splitlines()[-3:] == ['', '# %%', '1+1']
assert open(tmp_rmd).read().splitlines()[-4:] == [
'', '```{python}', '1+1', '```'
]
def test_pre_commit_hook_py_to_ipynb_and_md(tmpdir):
tmp_ipynb = str(tmpdir.join('nb with spaces.ipynb'))
tmp_py = str(tmpdir.join('nb with spaces.py'))
tmp_md = str(tmpdir.join('nb with spaces.md'))
nb = new_notebook(cells=[])
git = git_in_tmpdir(tmpdir)
git('init')
git('status')
hook = str(tmpdir.join('.git/hooks/pre-commit'))
with open(hook, 'w') as fp:
fp.write('#!/bin/sh\n'
'jupytext --from py:light --to ipynb --pre-commit\n'
'jupytext --from py:light --to md --pre-commit\n')
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
writef(nb, tmp_py)
assert os.path.isfile(tmp_py)
assert not os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_md)
git('add', 'nb with spaces.py')
git('status')
git('commit', '-m', 'created')
git('status')
assert 'nb with spaces.ipynb' in git('ls-tree', '-r', 'master',
'--name-only')
assert 'nb with spaces.md' in git('ls-tree', '-r', 'master', '--name-only')
assert os.path.isfile(tmp_ipynb)
assert os.path.isfile(tmp_md)
def test_manual_call_of_pre_commit_hook(tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_py = str(tmpdir.join('notebook.py'))
nb = new_notebook(cells=[])
os.chdir(str(tmpdir))
def system_in_tmpdir(*args):
return system(*args, cwd=str(tmpdir))
git = git_in_tmpdir(tmpdir)
git('init')
git('status')
def hook():
with mock.patch('jupytext.cli.system', system_in_tmpdir):
jupytext(['--to', 'py', '--pre-commit'])
writef(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_py)
git('add', 'notebook.ipynb')
git('status')
hook()
git('commit', '-m', 'created')
git('status')
assert 'notebook.py' in git('ls-tree', '-r', 'master', '--name-only')
assert os.path.isfile(tmp_py)
def write(cls, file_content, path):
"""
Write a notebook using Jupytext
"""
assert path.startswith('txt://')
nb = nbformat.reads(file_content, as_version=4)
jupytext.writef(nb, path[6:])
def test_pre_commit_hook(tmpdir):
tmp_ipynb = str(tmpdir.join('nb with spaces.ipynb'))
tmp_py = str(tmpdir.join('nb with spaces.py'))
nb = new_notebook(cells=[])
def git(*args):
print(system('git', *args, cwd=str(tmpdir)))
git('init')
git('status')
hook = str(tmpdir.join('.git/hooks/pre-commit'))
with open(hook, 'w') as fp:
fp.write('#!/bin/sh\n' 'jupytext --to py:light --pre-commit\n')
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
writef(nb, tmp_ipynb)
assert os.path.isfile(tmp_ipynb)
assert not os.path.isfile(tmp_py)
git('add', 'nb with spaces.ipynb')
git('status')
git('commit', '-m', 'created')
git('status')
assert os.path.isfile(tmp_py)
def strip_cells(nb_in, nb_out, the_string):
"""
Remove all cells with metadata['ctype']==the_string from
cell list.
Parameters
----------
nb_in: str
the path of the notebook to strip cells from
nb_out: str
the path of the notebook to write to
the_string: str
the cell type to strip
Returns
-------
obj
a new notebook object with cells stripped
"""
nb_obj = jp.readf(nb_in)
new_list = []
for the_cell in nb_obj['cells']:
if 'ctype' in the_cell['metadata']:
if the_cell['metadata']['ctype'] == the_string:
continue
new_list.append(the_cell)
nb_obj['cells'] = new_list
jp.writef(nb_obj, str(nb_out))
return nb_obj
def test_apply_black_and_sync_on_paired_notebook(tmpdir, nb_file):
# Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work
metadata = readf(nb_file).metadata
metadata['jupytext'] = {'formats': 'ipynb,py'}
assert 'language_info' in metadata
nb_org = new_notebook(cells=[new_code_cell('1 +1')],
metadata=metadata)
nb_black = new_notebook(cells=[new_code_cell('1 + 1')], metadata=metadata)
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_py = str(tmpdir.join('notebook.py'))
# Black in place
writef(nb_org, tmp_ipynb)
jupytext([tmp_ipynb, '--pipe', 'black', '--sync'])
nb_now = readf(tmp_ipynb)
compare(nb_black, nb_now)
assert 'language_info' in nb_now.metadata
nb_now = readf(tmp_py)
nb_now.metadata['jupytext'].pop('text_representation')
nb_black.metadata = {
key: nb_black.metadata[key]
for key in nb_black.metadata if key in _DEFAULT_NOTEBOOK_METADATA
}
compare(nb_black, nb_now)
def main(args=None):
parser = make_parser()
args = parser.parse_args(args)
nb_obj = jp.readf(args.pynotebook)
new_nb = nu.update_headers(nb_obj, lhead=args.lhead, chead=args.chead)
jp.writef(new_nb, args.pynotebook)
jupytext(args=[args.pynotebook, '--to', '..//ipynb', '--execute'])
Path(args.pynotebook).touch()
def test_cli_to_auto(nb_file, ext, tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_text = str(tmpdir.join('notebook' + ext))
nb = readf(nb_file)
writef(nb, tmp_ipynb)
jupytext(['--to', 'auto', tmp_ipynb])
nb2 = readf(tmp_text)
compare_notebooks(nb, nb2)
def save_notebook_as(notebook, nb_file, nb_dest, combine):
"""Save notebook to file, in desired format"""
if combine and os.path.isfile(nb_dest) and \
os.path.splitext(nb_dest)[1] == '.ipynb':
check_file_version(notebook, nb_file, nb_dest)
nb_outputs = readf(nb_dest)
combine_inputs_with_outputs(notebook, nb_outputs)
writef(notebook, nb_dest)
def main(notebook_file, cell_key, cell_value):
"""
python add_cellmeta.py python/assignment3_2019t1_solution.py ctype question
"""
nb_file = Path(notebook_file.name)
nb_obj = jp.readf(nb_file)
new_nb = nu.update_cell_meta(nb_obj, cell_key, cell_value)
jp.writef(new_nb, str(nb_file))
jupytext(args=[str(nb_file), "--sync"])
nb_file.touch()
def convert(pyfile):
"""
Convert a preprocessed string object into notebook file
"""
base = os.path.basename(pyfile)
root, ext = os.path.splitext(base)
target = os.path.join(to_dir, root + '.ipynb')
print('Converting: ' + pyfile + ' >>> ' + target)
nb = jupytext.readf(pyfile)
jupytext.writef(nb, target, fmt='notebook')
def assert_conversion_same_as_mirror(nb_file, fmt, mirror_name, compare_notebook=False):
dirname, basename = os.path.split(nb_file)
file_name, org_ext = os.path.splitext(basename)
fmt = long_form_one_format(fmt)
ext = fmt['extension']
mirror_file = os.path.join(dirname, '..', 'mirror', mirror_name, full_path(file_name, fmt))
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', False):
notebook = jupytext.readf(nb_file, fmt)
# it's better not to have Jupytext metadata in test notebooks:
if fmt == 'ipynb' and 'jupytext' in notebook.metadata: # pragma: no cover
notebook.metadata.pop('jupytext')
jupytext.writef(nb_file, fmt)
create_mirror_file_if_missing(mirror_file, notebook, fmt)
# Compare the text representation of the two notebooks
if compare_notebook:
nb_mirror = jupytext.readf(mirror_file)
compare(nb_mirror, notebook)
return
elif ext == '.ipynb':
notebook = jupytext.readf(mirror_file)
fmt.update({'extension': org_ext})
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', False):
actual = jupytext.writes(notebook, fmt)
with open(nb_file, encoding='utf-8') as fp:
expected = fp.read()
else:
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', False):
actual = jupytext.writes(notebook, fmt)
with open(mirror_file, encoding='utf-8') as fp:
expected = fp.read()
if not actual.endswith('\n'):
actual = actual + '\n'
compare(expected, actual)
# Compare the two notebooks
if ext != '.ipynb':
with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', False):
notebook = jupytext.readf(nb_file)
nb_mirror = jupytext.readf(mirror_file, fmt)
if fmt.get('format_name') == 'sphinx':
nb_mirror.cells = nb_mirror.cells[1:]
for cell in notebook.cells:
cell.metadata = {}
for cell in nb_mirror.cells:
cell.metadata = {}
compare_notebooks(notebook, nb_mirror, fmt)
combine_inputs_with_outputs(nb_mirror, notebook)
compare_notebooks(notebook, nb_mirror, fmt, compare_outputs=True)
def main(old_pynotebook, new_pynotebook):
"""
Usage: python strip_answers.py assignment3_2019t1_solution.py assignment3_2019t1_student.py
"""
old_pynotebook = Path(old_pynotebook.name)
new_pynotebook = Path(new_pynotebook.name)
nb_obj = jp.readf(old_pynotebook)
new_nb = nu.strip_answers(nb_obj, "answer")
jp.writef(new_nb, str(new_pynotebook))
jupytext(args=[str(new_pynotebook), "--sync", "--execute"])
Path(new_pynotebook).touch()
def test_write_notebook_does_not_change_it(nb_file, fmt, tmpdir):
nb_org = readf(nb_file)
nb_org_copied = deepcopy(nb_org)
ext = long_form_one_format(fmt, nb_org.metadata)['extension']
writes(nb_org, fmt)
compare(nb_org, nb_org_copied)
tmp_dest = str(tmpdir.join('notebook' + ext))
writef(nb_org, tmp_dest, fmt)
compare(nb_org, nb_org_copied)
def test_rename(tmpdir):
org_file = str(tmpdir.join('notebook.ipynb'))
new_file = str(tmpdir.join('new.ipynb'))
jupytext.writef(new_notebook(), org_file)
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
cm.rename_file('notebook.ipynb', 'new.ipynb')
assert os.path.isfile(new_file)
assert not os.path.isfile(org_file)
def test_cells_convert_all():
SAMPLE_DIR = Path('samples')
for py_path in list(SAMPLE_DIR.glob('**/*.py')):
print(py_path)
with py_path.open() as fin:
data = fin.read()
parsed = jupytext.reads(data, ext='.py', format_name='percent')
OUTPUT_IPYNB = py_path.parents[0] / Path(str(py_path.stem) + '.ipynb')
# assert OUTPUT_IPYNB.exists()
jupytext.writef(parsed, OUTPUT_IPYNB)
def test_paired_paths(nb_file, tmpdir, capsys):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
nb = readf(nb_file)
nb.metadata.setdefault('jupytext', {})['formats'] = 'ipynb,_light.py,_percent.py:percent'
writef(nb, tmp_ipynb)
jupytext(['--paired-paths', tmp_ipynb])
out, err = capsys.readouterr()
assert not err
formats = nb.metadata.get('jupytext', {}).get('formats')
assert set(out.splitlines()).union([tmp_ipynb]) == set(
[path for path, _ in paired_paths(tmp_ipynb, 'ipynb', formats)])
def test_cli_can_infer_jupytext_format(nb_file, ext, tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_text = str(tmpdir.join('notebook' + ext))
nb = readf(nb_file)
# Light format to Jupyter notebook
writef(nb, tmp_text)
jupytext(['--to', 'notebook', tmp_text])
nb2 = readf(tmp_ipynb)
compare_notebooks(nb, nb2)
# Percent format to Jupyter notebook
writef(nb, tmp_text, ext + ':percent')
jupytext(['--to', 'notebook', tmp_text])
nb2 = readf(tmp_ipynb)
compare_notebooks(nb, nb2)
def test_save_ipynb_with_jupytext_has_final_newline(tmpdir):
nb = new_notebook()
file_jupytext = str(tmpdir.join('jupytext.ipynb'))
file_nbformat = str(tmpdir.join('nbformat.ipynb'))
jupytext.writef(nb, file_jupytext)
with open(file_nbformat, 'w') as fp:
nbformat.write(nb, fp)
with open(file_jupytext) as fp:
text_jupytext = fp.read()
with open(file_nbformat) as fp:
text_nbformat = fp.read()
compare(text_nbformat, text_jupytext)
def test_rename_inconsistent_path(tmpdir):
org_file = str(tmpdir.join('notebook_suffix.ipynb'))
new_file = str(tmpdir.join('new.ipynb'))
jupytext.writef(
new_notebook(metadata={'jupytext': {
'formats': '_suffix.ipynb'
}}), org_file)
cm = jupytext.TextFileContentsManager()
cm.root_dir = str(tmpdir)
# Read notebook, and learn about its format
cm.get('notebook_suffix.ipynb')
with pytest.raises(HTTPError):
cm.rename_file('notebook_suffix.ipynb', 'new.ipynb')
assert not os.path.isfile(new_file)
assert os.path.isfile(org_file)
def test_pre_commit_hook_sync_black_flake8(tmpdir, nb_file):
# Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work
metadata = readf(nb_file).metadata
def git(*args):
print(system('git', *args, cwd=str(tmpdir)))
git('init')
git('status')
hook = str(tmpdir.join('.git/hooks/pre-commit'))
with open(hook, 'w') as fp:
fp.write(
'#!/bin/sh\n'
'# Pair ipynb notebooks to a python file, reformat content with black, and run flake8\n'
'# Note: this hook only acts on ipynb files. When pulling, run jupytext --sync manually to '
'update the ipynb file.\n'
'jupytext --pre-commit --from ipynb --set-formats ipynb,py --sync --pipe black --check flake8\n'
)
st = os.stat(hook)
os.chmod(hook, st.st_mode | stat.S_IEXEC)
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_py = str(tmpdir.join('notebook.py'))
nb = new_notebook(cells=[new_code_cell(source='1+ 1')],
metadata=metadata)
writef(nb, tmp_ipynb)
git('add', 'notebook.ipynb')
git('status')
git('commit', '-m', 'created')
git('status')
assert os.path.isfile(tmp_py)
assert os.path.isfile(tmp_ipynb)
with open(tmp_py) as fp:
assert fp.read().splitlines()[-1] == '1 + 1'
nb = new_notebook(
cells=[new_code_cell(source='"""trailing \nwhitespace"""')],
metadata=metadata)
writef(nb, tmp_ipynb)
git('add', 'notebook.ipynb')
git('status')
with pytest.raises(SystemExit): # not flake8
git('commit', '-m', 'created')
def test_wildcard(tmpdir):
nb1_ipynb = str(tmpdir.join('nb1.ipynb'))
nb2_ipynb = str(tmpdir.join('nb2.ipynb'))
nb1_py = str(tmpdir.join('nb1.py'))
nb2_py = str(tmpdir.join('nb2.py'))
writef(new_notebook(metadata={'notebook': 1}), nb1_ipynb)
writef(new_notebook(metadata={'notebook': 2}), nb2_ipynb)
os.chdir(str(tmpdir))
jupytext(['nb*.ipynb', '--to', 'py'])
assert os.path.isfile(nb1_py)
assert os.path.isfile(nb2_py)
with pytest.raises(IOError):
jupytext(['nb3.ipynb', '--to', 'py'])
def test_cells_convert():
SAMPLE_DIR = os.path.join(os.getcwd(), 'samples')
INPUT_FNAME = 'hello_cells.py'
BASENAME, _ = os.path.splitext(INPUT_FNAME)
INPUT_PY = os.path.join(SAMPLE_DIR, INPUT_FNAME)
print("Input file:", INPUT_PY)
assert os.path.exists(INPUT_PY)
OUTPUT_IPYNB = os.path.join(SAMPLE_DIR, BASENAME + ".ipynb")
print("Output file: ", OUTPUT_IPYNB)
if os.path.exists(OUTPUT_IPYNB):
os.remove(OUTPUT_IPYNB)
with open(INPUT_PY) as fin:
data = fin.read()
parsed = jupytext.reads(data, ext='.py', format_name='percent')
jupytext.writef(parsed, OUTPUT_IPYNB)
def test_sync(nb_file, tmpdir):
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
tmp_py = str(tmpdir.join('notebook.py'))
tmp_rmd = str(tmpdir.join('notebook.Rmd'))
nb = readf(nb_file)
writef(nb, tmp_ipynb)
# Test that sync fails when notebook is not paired
with pytest.raises(ValueError) as info:
jupytext(['--sync', tmp_ipynb])
assert 'is not a paired notebook' in str(info)
# Now with a pairing information
nb.metadata.setdefault('jupytext', {})['formats'] = 'py,Rmd,ipynb'
writef(nb, tmp_ipynb)
# Test that missing files are created
jupytext(['--sync', tmp_ipynb])
assert os.path.isfile(tmp_py)
compare_notebooks(nb, readf(tmp_py))
assert os.path.isfile(tmp_rmd)
compare_notebooks(nb, readf(tmp_rmd), 'Rmd')
# Now we keep only the first four cells and save to Rmd
nb.cells = nb.cells[:4]
writef(nb, tmp_rmd, 'Rmd')
jupytext(['--sync', tmp_ipynb])
nb2 = readf(tmp_ipynb)
compare_notebooks(nb, nb2, 'Rmd', compare_outputs=True)
# Now we keep only the first two cells and save to py
nb.cells = nb.cells[:4]
writef(nb, tmp_py, 'py')
jupytext(['--sync', tmp_ipynb])
nb2 = readf(tmp_ipynb)
compare_notebooks(nb, nb2, compare_outputs=True)
# Finally we recreate the ipynb
os.remove(tmp_ipynb)
time.sleep(0.1)
jupytext(['--sync', tmp_py])
nb2 = readf(tmp_ipynb)
compare_notebooks(nb, nb2)
# ipynb must be older than py file, otherwise our Contents Manager will complain
assert os.path.getmtime(tmp_ipynb) < os.path.getmtime(tmp_py)
def test_rst2md(tmpdir):
tmp_py = str(tmpdir.join('notebook.py'))
tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
# Write notebook in sphinx format
nb = new_notebook(cells=[new_markdown_cell('A short sphinx notebook'),
new_markdown_cell(':math:`1+1`')])
writef(nb, tmp_py, 'py:sphinx')
jupytext([tmp_py, '--from', 'py:sphinx', '--to', 'ipynb',
'--opt', 'rst2md=True', '--opt', 'cell_metadata_filter=-all'])
assert os.path.isfile(tmp_ipynb)
nb = readf(tmp_ipynb)
assert nb.metadata['jupytext']['cell_metadata_filter'] == '-all'
assert nb.metadata['jupytext']['rst2md'] is False
# Was rst to md conversion effective?
assert nb.cells[2].source == '$1+1$'
