def test_auto_in_formats(nb_file): nb = read(nb_file) nb.metadata["jupytext"] = {"formats": "ipynb,auto:percent"} fmt = auto_ext_from_metadata(nb.metadata)[1:] + ":percent" expected_formats = "ipynb," + fmt text = writes(nb, "ipynb") assert "auto" not in text nb2 = reads(text, "ipynb") assert nb2.metadata["jupytext"]["formats"] == expected_formats text = writes(nb, "auto:percent") assert "auto" not in text nb2 = reads(text, fmt) assert nb2.metadata["jupytext"]["formats"] == expected_formats del nb.metadata["language_info"] del nb.metadata["kernelspec"] with pytest.raises(JupytextFormatError): writes(nb, "ipynb") with pytest.raises(JupytextFormatError): writes(nb, "auto:percent")
def test_pipe_nbconvert_execute_sync(tmpdir): tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_py = str(tmpdir.join("notebook.py")) with open(tmp_py, "w") as fp: fp.write("""1 + 2 """) jupytext(args=[ tmp_py, "--set-formats", "py,ipynb", "--sync", "--pipe-fmt", "ipynb", "--pipe", "jupyter nbconvert --stdin --stdout --to notebook --execute", ]) nb = read(tmp_ipynb) assert len(nb.cells) == 1 assert nb.cells[0].outputs[0]["data"] == {"text/plain": "3"}
def test_cell_markers_in_contents_manager_does_not_impact_light_format(tmpdir): tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_py = str(tmpdir.join("notebook.py")) cm = jupytext.TextFileContentsManager() cm.root_dir = str(tmpdir) cm.cell_markers = "'''" nb = new_notebook( cells=[new_code_cell("1 + 1"), new_markdown_cell("a\nlong\ncell")], metadata={ "jupytext": { "formats": "ipynb,py", "notebook_metadata_filter": "-all" } }, ) with pytest.warns(UserWarning, match="Ignored cell markers"): cm.save(model=notebook_model(nb), path="notebook.ipynb") assert os.path.isfile(tmp_ipynb) assert os.path.isfile(tmp_py) with open(tmp_py) as fp: text = fp.read() compare( text, """1 + 1 # a # long # cell """, ) nb2 = jupytext.read(tmp_py) compare_notebooks(nb, nb2)
def test_auto_in_formats(nb_file): nb = read(nb_file) nb.metadata['jupytext'] = {'formats': 'ipynb,auto:percent'} fmt = auto_ext_from_metadata(nb.metadata)[1:] + ':percent' expected_formats = 'ipynb,' + fmt text = writes(nb, 'ipynb') assert 'auto' not in text nb2 = reads(text, 'ipynb') assert nb2.metadata['jupytext']['formats'] == expected_formats text = writes(nb, 'auto:percent') assert 'auto' not in text nb2 = reads(text, fmt) assert nb2.metadata['jupytext']['formats'] == expected_formats del nb.metadata['language_info'] del nb.metadata['kernelspec'] with pytest.raises(JupytextFormatError): writes(nb, 'ipynb') with pytest.raises(JupytextFormatError): writes(nb, 'auto:percent')
def test_load_save_rename_non_ascii_path(nb_file, tmpdir): tmp_ipynb = u'notebôk.ipynb' tmp_nbpy = u'notebôk.nb.py' cm = jupytext.TextFileContentsManager() cm.default_jupytext_formats = 'ipynb,.nb.py' tmpdir = u'' + str(tmpdir) cm.root_dir = tmpdir # open ipynb, save nb.py, reopen nb = jupytext.read(nb_file) cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy) nbpy = cm.get(tmp_nbpy) compare_notebooks(nb, nbpy['content']) # open ipynb nbipynb = cm.get(tmp_ipynb) compare_notebooks(nb, nbipynb['content']) # save ipynb cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) # rename notebôk.nb.py to nêw.nb.py cm.rename(tmp_nbpy, u'nêw.nb.py') assert not os.path.isfile(os.path.join(tmpdir, tmp_ipynb)) assert not os.path.isfile(os.path.join(tmpdir, tmp_nbpy)) assert os.path.isfile(os.path.join(tmpdir, u'nêw.ipynb')) assert os.path.isfile(os.path.join(tmpdir, u'nêw.nb.py')) # rename nêw.ipynb to notebôk.ipynb cm.rename(u'nêw.ipynb', tmp_ipynb) assert os.path.isfile(os.path.join(tmpdir, tmp_ipynb)) assert os.path.isfile(os.path.join(tmpdir, tmp_nbpy)) assert not os.path.isfile(os.path.join(tmpdir, u'nêw.ipynb')) assert not os.path.isfile(os.path.join(tmpdir, u'nêw.nb.py'))
def test_default_cell_markers_in_contents_manager(tmpdir): tmp_ipynb = str(tmpdir.join('notebook.ipynb')) tmp_py = str(tmpdir.join('notebook.py')) cm = jupytext.TextFileContentsManager() cm.root_dir = str(tmpdir) cm.default_cell_markers = "'''" nb = new_notebook( cells=[new_code_cell('1 + 1'), new_markdown_cell('a\nlong\ncell')], metadata={ 'jupytext': { 'formats': 'ipynb,py:percent', 'notebook_metadata_filter': '-all' } }) cm.save(model=dict(type='notebook', content=nb), path='notebook.ipynb') assert os.path.isfile(tmp_ipynb) assert os.path.isfile(tmp_py) with open(tmp_py) as fp: text = fp.read() compare(text, """# %% 1 + 1 # %% [markdown] ''' a long cell ''' """) nb2 = jupytext.read(tmp_py) compare_notebooks(nb, nb2)
def test_save_to_light_percent_sphinx_format(nb_file, tmpdir): tmp_ipynb = 'notebook.ipynb' tmp_lgt_py = 'notebook.lgt.py' tmp_pct_py = 'notebook.pct.py' tmp_spx_py = 'notebook.spx.py' cm = jupytext.TextFileContentsManager() cm.root_dir = str(tmpdir) nb = jupytext.read(nb_file) nb['metadata']['jupytext'] = { 'formats': 'ipynb,.pct.py:percent,.lgt.py:light,.spx.py:sphinx' } # save to ipynb and three python flavors cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) # read files with open(str(tmpdir.join(tmp_pct_py))) as stream: assert read_format_from_metadata(stream.read(), '.py') == 'percent' with open(str(tmpdir.join(tmp_lgt_py))) as stream: assert read_format_from_metadata(stream.read(), '.py') == 'light' with open(str(tmpdir.join(tmp_spx_py))) as stream: assert read_format_from_metadata(stream.read(), '.py') == 'sphinx' model = cm.get(path=tmp_pct_py) compare_notebooks(nb, model['content']) model = cm.get(path=tmp_lgt_py) compare_notebooks(nb, model['content']) model = cm.get(path=tmp_spx_py) # (notebooks not equal as we insert %matplotlib inline in sphinx) model = cm.get(path=tmp_ipynb) compare_notebooks(nb, model['content'])
def exe_check_nb_fname(nb_fname, wd=None): """ Execute notebook, return error message or 'ok' Parameters ---------- nb_fname : str Filename of notebook to execute. wd : None or str Directory in which to execute notebook. None means use directory containing `nb_fname` Returns ------- err_msg : str Error message or 'ok' for no error. """ wd = op.dirname(nb_fname) if wd is None else wd nb = jupytext.read(nb_fname) try: executenb(nb, cwd=wd, kernel_name=nb2kernel_name(nb)) except CellExecutionError as e: return str(e) return 'ok'
def test_outdated_text_notebook(nb_file, tmpdir): # 1. write py ipynb tmp_ipynb = u'notebook.ipynb' tmp_nbpy = u'notebook.py' cm = jupytext.TextFileContentsManager() cm.default_jupytext_formats = 'py,ipynb' cm.outdated_text_notebook_margin = 0 cm.root_dir = str(tmpdir) # open ipynb, save py, reopen nb = jupytext.read(nb_file) cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy) model_py = cm.get(tmp_nbpy, load_alternative_format=False) model_ipynb = cm.get(tmp_ipynb, load_alternative_format=False) # 2. check that time of ipynb <= py assert model_ipynb['last_modified'] <= model_py['last_modified'] # 3. wait some time time.sleep(0.5) # 4. touch ipynb with open(str(tmpdir.join(tmp_ipynb)), 'a'): os.utime(str(tmpdir.join(tmp_ipynb)), None) # 5. test error with pytest.raises(HTTPError): cm.get(tmp_nbpy) # 6. test OK with cm.outdated_text_notebook_margin = 1.0 cm.get(tmp_nbpy) # 7. test OK with cm.outdated_text_notebook_margin = float("inf") cm.get(tmp_nbpy)
def test_pair_plain_script(py_file, tmpdir): tmp_py = 'notebook.py' tmp_ipynb = 'notebook.ipynb' cm = jupytext.TextFileContentsManager() cm.root_dir = str(tmpdir) # open py file, pair, save with cm nb = jupytext.read(py_file) nb.metadata['jupytext']['formats'] = 'ipynb,py:hydrogen' cm.save(model=dict(type='notebook', content=nb), path=tmp_py) assert os.path.isfile(str(tmpdir.join(tmp_py))) assert os.path.isfile(str(tmpdir.join(tmp_ipynb))) # Make sure we've not changed the script with open(py_file) as fp: script = fp.read() with open(str(tmpdir.join(tmp_py))) as fp: script2 = fp.read() compare(script, script2) # reopen py file with the cm nb2 = cm.get(tmp_py)['content'] compare_notebooks(nb, nb2) assert nb2.metadata['jupytext']['formats'] == 'ipynb,py:hydrogen' # remove the pairing and save del nb.metadata['jupytext']['formats'] cm.save(model=dict(type='notebook', content=nb), path=tmp_py) # reopen py file with the cm nb2 = cm.get(tmp_py)['content'] compare_notebooks(nb, nb2) assert 'formats' not in nb2.metadata['jupytext']
def render_rmd(input_file: str, output_file: str, params: dict = None): """ Wrapper function to render an Rmarkdown document with the R `rmarkdown` package and convert it to HTML using pandoc and a custom template. Parameters ---------- input_file path to input (Rmd) file output_file path to output (html) file params Dictionary that will be passed to `params` arg of `rmarkdown::render`. See https://bookdown.org/yihui/rmarkdown/parameterized-reports.html for more details. """ # Directory the notebook is located in. Will be used as additional resource path for pandoc. nb_dir = os.path.abspath(os.path.dirname(input_file)) with TemporaryDirectory() as tmp_dir: with TemporaryDirectory() as tmp_dir_nb_converted: # Create this file manually with given name # so that pandoc gracefully falls back to the # file name if no title is specified within the file. (#17) tmp_nb_converted = os.path.join( tmp_dir_nb_converted, os.path.splitext(os.path.basename(input_file))[0] + ".Rmd", ) if not input_file.endswith(".Rmd"): nb = jtx.read(input_file) jtx.write(nb, tmp_nb_converted) else: copyfile(input_file, tmp_nb_converted) md_file = _run_rmarkdown(tmp_nb_converted, tmp_dir, params) run_pandoc(md_file, output_file, res_path=[nb_dir, tmp_dir])
def test_load_save_rename_nbpy_default_config(nb_file, tmpdir): tmp_ipynb = 'notebook.ipynb' tmp_nbpy = 'notebook.nb.py' cm = jupytext.TextFileContentsManager() cm.default_jupytext_formats = 'ipynb,.nb.py' cm.root_dir = str(tmpdir) # open ipynb, save nb.py, reopen nb = jupytext.read(nb_file) cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy) nbpy = cm.get(tmp_nbpy) compare_notebooks(nbpy['content'], nb) # open ipynb nbipynb = cm.get(tmp_ipynb) compare_notebooks(nbipynb['content'], nb) # save ipynb cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) # rename notebook.nb.py to new.nb.py cm.rename(tmp_nbpy, 'new.nb.py') assert not os.path.isfile(str(tmpdir.join(tmp_ipynb))) assert not os.path.isfile(str(tmpdir.join(tmp_nbpy))) assert os.path.isfile(str(tmpdir.join('new.ipynb'))) assert os.path.isfile(str(tmpdir.join('new.nb.py'))) # rename new.ipynb to notebook.ipynb cm.rename('new.ipynb', tmp_ipynb) assert os.path.isfile(str(tmpdir.join(tmp_ipynb))) assert os.path.isfile(str(tmpdir.join(tmp_nbpy))) assert not os.path.isfile(str(tmpdir.join('new.ipynb'))) assert not os.path.isfile(str(tmpdir.join('new.nb.py')))
def test_sync_pipe_config(tmpdir): """Sync a notebook to a script paired in a tree, and reformat the markdown cells using pandoc""" tmpdir.join("jupytext.toml").write( """# By default, the notebooks in this repository are in the notebooks subfolder # and they are paired to scripts in the script subfolder. default_jupytext_formats = "notebooks///ipynb,scripts///py:percent" """ ) nb_file = tmpdir.mkdir("notebooks").join("wrap_markdown.ipynb") long_text = "This is a " + ("very " * 24) + "long sentence." assert len(long_text) > 100 nb = new_notebook(cells=[new_markdown_cell(long_text)]) write(nb, str(nb_file)) jupytext( [ "--sync", "--pipe-fmt", "ipynb", "--pipe", "pandoc --from ipynb --to ipynb --atx-headers", str(nb_file), ] ) py_text = tmpdir.join("scripts").join("wrap_markdown.py").read() assert "This is a very very" in py_text for line in py_text.splitlines(): assert len(line) <= 79 nb = read(nb_file, as_version=4) text = nb.cells[0].source assert len(text.splitlines()) == 3 assert text != long_text
def test_execute_in_subfolder(tmpdir): tmpdir.mkdir('subfolder') tmp_csv = str(tmpdir.join('subfolder', 'inputs.csv')) tmp_py = str(tmpdir.join('subfolder', 'notebook.py')) tmp_ipynb = str(tmpdir.join('subfolder', 'notebook.ipynb')) with open(tmp_csv, 'w') as fp: fp.write("1\n2\n") with open(tmp_py, 'w') as fp: fp.write("""import ast with open('inputs.csv') as fp: text = fp.read() sum(ast.literal_eval(line) for line in text.splitlines()) """) jupytext(args=[tmp_py, '--to', 'ipynb', '--execute']) nb = read(tmp_ipynb) assert len(nb.cells) == 3 assert nb.cells[2].outputs[0]['data'] == {'text/plain': '3'}
def test_execute_in_subfolder(tmpdir, caplog, capsys): tmpdir.mkdir("subfolder") tmp_csv = str(tmpdir.join("subfolder", "inputs.csv")) tmp_py = str(tmpdir.join("subfolder", "notebook.py")) tmp_ipynb = str(tmpdir.join("subfolder", "notebook.ipynb")) with open(tmp_csv, "w") as fp: fp.write("1\n2\n") with open(tmp_py, "w") as fp: fp.write("""import ast with open('inputs.csv') as fp: text = fp.read() sum(ast.literal_eval(line) for line in text.splitlines()) """) jupytext(args=[tmp_py, "--to", "ipynb", "--execute"]) nb = read(tmp_ipynb) assert len(nb.cells) == 3 assert nb.cells[2].outputs[0]["data"] == {"text/plain": "3"}
def test_jupytext(tmpdir): path_jupytext_dir = op.join(this_folder, 'site', 'content', 'tests', 'jupytext') jupytext_files = { '.md': op.join(path_jupytext_dir, 'jupytext_md.md'), '.ipynb': op.join(path_jupytext_dir, 'jupytext_ipynb.ipynb'), '.py': op.join(path_jupytext_dir, 'jupytext_py.py'), '.Rmd': op.join(path_jupytext_dir, 'jupytext_Rmd.Rmd') } jupytext_html = {} for ext, ifile in jupytext_files.items(): ntbk = jpt.read(ifile) html, resources = page_html(ntbk, execute_dir=path_jupytext_dir) jupytext_html[ext] = html assert "tag_hide_input" in html assert "This message should display: 4" in html # Make sure all of the jupytext files in different formats are the same base_ext = ".ipynb" for compare_ext in [".md", ".py", ".Rmd"]: assert jupytext_html[base_ext] == jupytext_html[compare_ext]
def test_rst2md(tmpdir): tmp_py = str(tmpdir.join('notebook.py')) tmp_ipynb = str(tmpdir.join('notebook.ipynb')) # Write notebook in sphinx format nb = new_notebook(cells=[ new_markdown_cell('A short sphinx notebook'), new_markdown_cell(':math:`1+1`') ]) write(nb, tmp_py, fmt='py:sphinx') jupytext([ tmp_py, '--from', 'py:sphinx', '--to', 'ipynb', '--opt', 'rst2md=True', '--opt', 'cell_metadata_filter=-all' ]) assert os.path.isfile(tmp_ipynb) nb = read(tmp_ipynb) assert nb.metadata['jupytext']['cell_metadata_filter'] == '-all' assert nb.metadata['jupytext']['rst2md'] is False # Was rst to md conversion effective? assert nb.cells[2].source == '$1+1$'
def test_page_standalone(tmpdir): path_ipynb = op.join(this_folder, 'site', 'content', 'tests', 'notebooks.ipynb') path_out = op.join(tmpdir.dirpath(), '.') ntbk = jpt.read(path_ipynb) html, resources = page_html(ntbk, execute_dir=op.dirname(path_ipynb)) custom_css = """ h1 { font-size: REALLYBIG; } """ custom_js = """ console.log("OMG") """ path_html = write_page(html, path_out, resources, standalone=True, custom_css=custom_css, custom_js=custom_js) with open(path_html, 'r') as ff: html = ff.read() assert "<!DOCTYPE html>" in html assert custom_css in html assert custom_js in html
def test_load_save_rename_notebook_with_dot(nb_file, tmpdir): tmp_ipynb = '1.notebook.ipynb' tmp_nbpy = '1.notebook.py' cm = jupytext.TextFileContentsManager() cm.default_jupytext_formats = 'ipynb,py' cm.root_dir = str(tmpdir) # open ipynb, save nb.py, reopen nb = jupytext.read(nb_file) cm.save(model=dict(type='notebook', content=nb), path=tmp_nbpy) nbpy = cm.get(tmp_nbpy) compare_notebooks(nb, nbpy['content']) # save ipynb cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) # rename py cm.rename(tmp_nbpy, '2.new_notebook.py') assert not os.path.isfile(str(tmpdir.join(tmp_ipynb))) assert not os.path.isfile(str(tmpdir.join(tmp_nbpy))) assert os.path.isfile(str(tmpdir.join('2.new_notebook.ipynb'))) assert os.path.isfile(str(tmpdir.join('2.new_notebook.py')))
def test_save_in_auto_extension_local(nb_file, tmpdir): # load notebook nb = jupytext.read(nb_file) nb.metadata.setdefault('jupytext', {})['formats'] = 'ipynb,auto:percent' auto_ext = auto_ext_from_metadata(nb.metadata) tmp_ipynb = 'notebook.ipynb' tmp_script = 'notebook' + auto_ext # create contents manager with default load format as percent cm = jupytext.TextFileContentsManager() cm.root_dir = str(tmpdir) # save notebook cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) # check that text representation exists, and is in percent format with open(str(tmpdir.join(tmp_script))) as stream: assert read_format_from_metadata(stream.read(), auto_ext) == 'percent' # reload and compare with original notebook model = cm.get(path=tmp_script) compare_notebooks(nb, model['content'])
def test_sync_with_pre_commit_hook(tmpdir): # Init git and create a pre-commit hook git = git_in_tmpdir(tmpdir) hook = str(tmpdir.join(".git/hooks/pre-commit")) with open(hook, "w") as fp: fp.write("#!/bin/sh\n" "jupytext --sync --pre-commit\n") st = os.stat(hook) os.chmod(hook, st.st_mode | stat.S_IEXEC) # Create a notebook that is not paired tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_md = str(tmpdir.join("notebook.md")) nb = new_notebook(cells=[new_markdown_cell("A short notebook")]) write(nb, tmp_ipynb) assert os.path.isfile(tmp_ipynb) assert not os.path.isfile(tmp_md) git("add", "notebook.ipynb") git("status") git("commit", "-m", "created") git("status") assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only") assert "notebook.md" not in git("ls-tree", "-r", "master", "--name-only") assert os.path.isfile(tmp_ipynb) assert not os.path.exists(tmp_md) # Pair the notebook jupytext(["--set-formats", "ipynb,md", tmp_ipynb]) # Remove the md file (it will be regenerated by the pre-commit hook) os.remove(tmp_md) # Commit the ipynb file git("add", "notebook.ipynb") git("status") git("commit", "-m", "paired") git("status") # The pre-commit script should have created and committed the md file assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only") assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only") assert os.path.isfile(tmp_md) nb_md = read(tmp_md) compare_notebooks(nb_md, nb) # Edit the md file with open(tmp_md) as fp: md_text = fp.read() with open(tmp_md, "w") as fp: fp.write(md_text.replace("A short notebook", "Notebook was edited")) # commit the md file git("add", "notebook.md") git("status") git("commit", "-m", "edited md") git("status") # The pre-commit script should have sync and committed the ipynb file assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only") assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only") nb = read(tmp_ipynb) compare(nb.cells, [new_markdown_cell("Notebook was edited")]) # create and commit a jpg file tmp_jpg = str(tmpdir.join("image.jpg")) with open(tmp_jpg, "wb") as fp: fp.write(b"") git("add", "image.jpg") git("commit", "-m", "added image")
def test_as_version_has_appropriate_type(): with pytest.raises(TypeError): jupytext.read("script.py", "py:percent")
def test_pre_commit_hook_sync_black_nbstripout( tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev, notebook_with_outputs, ): """Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout.""" pre_commit_config_yaml = f""" repos: - repo: {jupytext_repo_root} rev: {jupytext_repo_rev} hooks: - id: jupytext args: [--sync, --pipe, black] additional_dependencies: - black==20.8b1 # Matches hook - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black - repo: https://github.com/kynan/nbstripout rev: 0.3.9 hooks: - id: nbstripout """ tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) tmpdir.join(".jupytext.toml").write('default_jupytext_formats = "ipynb,py:percent"') tmp_repo.git.add(".jupytext.toml") tmp_repo.index.commit("pair notebooks") # write a test notebook write(notebook_with_outputs, "test.ipynb") # try to commit it, should fail because # 1. the py version hasn't been added # 2. the first cell is '1+1' which is not black compliant # 3. the notebook has outputs tmp_repo.git.add("test.ipynb") with pytest.raises( HookExecutionError, match="files were modified by this hook", ): tmp_repo.index.commit("failing") # Add the two files tmp_repo.git.add("test.ipynb") tmp_repo.git.add("test.py") # now the commit will succeed tmp_repo.index.commit("passing") assert "test.ipynb" in tmp_repo.tree() assert "test.py" in tmp_repo.tree() # the first cell was reformatted nb = read("test.ipynb") assert nb.cells[0].source == "1 + 1" # the ipynb file has no outputs assert not nb.cells[0].outputs
'func_render_all', 'func_render_instancing', 'func_accel_consistency', 'func_py_custom_material', 'func_py_custom_renderer', 'func_distributed_rendering', 'func_distributed_rendering_ext', 'func_error_handling', 'func_obj_loader_consistency', 'func_render_instancing', 'func_serial_consistency', 'func_update_asset', 'perf_accel', 'perf_obj_loader', 'perf_serial' ] # Execute tests for test in tests: print(Fore.GREEN + "Running test [name='{}']".format(test) + Style.RESET_ALL, flush=True) # Read the requested notebook nb = jupytext.read(os.path.join(base_path, test + '.py')) # Execute the notebook ep = ExecutePreprocessor(timeout=600) ep.preprocess(nb, {'metadata': {'path': base_path}}) # Write result with open(os.path.join(args.output_dir, test + '.ipynb'), mode='w', encoding='utf-8') as f: nbformat.write(nb, f) # Merge executed notebooks print(Fore.GREEN + "Merging notebooks" + Style.RESET_ALL) notebook_paths = [ os.path.join(args.output_dir, test + '.ipynb') for test in tests
def test_read_wrong_ext(tmpdir, nb_file='notebook.ext'): nb_file = tmpdir.join(nb_file) nb_file.write('{}') with pytest.raises(JupytextFormatError): jupytext.read(str(nb_file))
def test_format_with_extension_change(tmp_nbs): dag = DAGSpec('pipeline.yaml').to_dag().render() dag['load'].source.format(fmt='ipynb') assert not Path('load.py').exists() assert jupytext.read('load.ipynb')
def convert_to_myst(file, dest_path): # Read a notebook from a file ntbk = jupytext.read(file, fmt='ipynb') # Write notebook to md file jupytext.write(ntbk, dest_path.joinpath(file.stem + '.md'), fmt='md')
def to_notebook(path): notebook = jupytext.read(path) jupytext.write(notebook, replace_extension(path, ".ipynb"))
def build_page(path_ntbk, path_html_output, path_media_output=None, execute=False, path_template=None, verbose=False, kernel_name=None): """Build the HTML for a single notebook page. Inputs ====== path_ntbk : string The path to a notebook or text file we want to convert. If a text file, then Jupytext will be used to convert into a notebook. This will also cause the notebook to be *run* (e.g. execute=True). path_html_output : string The path to the folder where the HTML will be output. path_media_output : string | None If a string, the path to where images should be extracted. If None, images will be embedded in the HTML. execute : bool Whether to execute the notebook before converting path_template : string A path to the template used in conversion. kernel_name : string The name of the kernel to use if we execute notebooks. """ ######################################## # Load in the notebook notebook_name, suff = op.splitext(op.basename(path_ntbk)) is_raw_markdown_file = False if suff in ['.md', '.markdown']: # If it's a markdown file, we need to check whether it's a jupytext format with open(path_ntbk, 'r') as ff: lines = ff.readlines() yaml_lines, content = _split_yaml(lines) yaml = YAML().load(''.join(yaml_lines)) if (yaml is not None) and yaml.get('jupyter', {}).get('jupytext'): # If we have jupytext metadata, then use it to read the markdown file ntbk = jpt.reads(''.join(lines), 'md') else: # Otherwise, create an empty notebook and add all of the file contents as a markdown file is_raw_markdown_file = True ntbk = nbf.v4.new_notebook() ntbk['cells'].append( nbf.v4.new_markdown_cell(source=''.join(content))) else: # If it's not markdown, we assume it's either ipynb or a jupytext format ntbk = jpt.read(path_ntbk) if _is_jupytext_file(ntbk): execute = True ######################################## # Notebook cleaning # Minor edits to cells _clean_markdown_cells(ntbk) ############################################# # Conversion to HTML # create a configuration object that changes the preprocessors c = Config() c.FilesWriter.build_directory = path_html_output # Remove cell elements using tags c.TagRemovePreprocessor.remove_cell_tags = ("remove_cell", "removecell") c.TagRemovePreprocessor.remove_all_outputs_tags = ('remove_output', ) c.TagRemovePreprocessor.remove_input_tags = ('remove_input', ) # Remove any cells that are *only* whitespace c.RegexRemovePreprocessor.patterns = ["\\s*\\Z"] c.HTMLExporter.preprocessors = [ 'nbconvert.preprocessors.TagRemovePreprocessor', 'nbconvert.preprocessors.RegexRemovePreprocessor', # So the images are written to disk 'nbconvert.preprocessors.ExtractOutputPreprocessor', # Wrap cells in Jekyll raw tags _RawCellPreprocessor, ] # The text used as the text for anchor links. # TEMPORATILY Set to empty since we'll use anchor.js for the links # Once https://github.com/jupyter/nbconvert/pull/1101 is fixed # set to '<i class="fas fa-link"> </i>' c.HTMLExporter.anchor_link_text = ' ' # Excluding input/output prompts c.HTMLExporter.exclude_input_prompt = True c.HTMLExporter.exclude_output_prompt = True # Excution of the notebook if we wish if execute is True: ntbk = run_ntbk(ntbk, op.dirname(path_ntbk)) # Define the path to images and then the relative path to where they'll originally be placed if isinstance(path_media_output, str): path_media_output_rel = op.relpath(path_media_output, path_html_output) # Generate HTML from our notebook using the template output_resources = { 'output_files_dir': path_media_output_rel, 'unique_key': notebook_name } exp = HTMLExporter(template_file=path_template, config=c) html, resources = exp.from_notebook_node(ntbk, resources=output_resources) html = '<main class="jupyter-page">\n' + html + '\n</main>\n' # Now write the markdown and resources writer = FilesWriter(config=c) writer.write(html, resources, notebook_name=notebook_name) # Add the frontmatter to the yaml file in case it's wanted if is_raw_markdown_file and len(yaml_lines) > 0: with open(op.join(path_html_output, notebook_name + '.html'), 'r') as ff: md_lines = ff.readlines() md_lines.insert(0, '---\n') for iline in yaml_lines[::-1]: md_lines.insert(0, iline + '\n') md_lines.insert(0, '---\n') with open(op.join(path_html_output, notebook_name + '.html'), 'w') as ff: ff.writelines(md_lines) if verbose: print("Finished writing notebook to {}".format(path_html_output))
def build_book(path_book, path_toc_yaml=None, path_ssg_config=None, path_template=None, local_build=False, execute=False, overwrite=False): """Build the HTML for a book using its TOC and a content folder. Parameters ---------- path_book : str Path to the root of the book repository path_toc_yaml : str | None Path to the Table of Contents YAML file path_ssg_config : str | None Path to the Jekyll configuration file path_template : str | None Path to the template nbconvert uses to build HTML files local_build : bool Specify you are building site locally for later upload execute : bool Whether to execute notebooks before converting to HTML overwrite : bool Whether to overwrite existing HTML files """ if not op.isdir(path_book): raise _error("Could not find a Jupyter Book at the given location.\n" "Double-check the path you've provided:\n" "\n" f"{path_book}") _check_book_versions(path_book) PATH_IMAGES_FOLDER = op.join(path_book, '_build', 'images') BUILD_FOLDER = op.join(path_book, BUILD_FOLDER_NAME) ############################################### # Read in textbook configuration # Load the yaml for this site with open(path_ssg_config, 'r') as ff: site_yaml = yaml.safe_load(ff.read()) CONTENT_FOLDER_NAME = site_yaml.get('content_folder_name').strip('/') PATH_CONTENT_FOLDER = op.join(path_book, CONTENT_FOLDER_NAME) # Load the textbook yaml for this site if not op.exists(path_toc_yaml): raise _error("No toc.yml file found, please create one at `{}`".format( path_toc_yaml)) with open(path_toc_yaml, 'r') as ff: toc = yaml.safe_load(ff.read()) # Drop divider items and non-linked pages in the sidebar, un-nest sections toc = _prepare_toc(toc) ################################################ # Generating the Jekyll files for all content n_skipped_files = 0 n_built_files = 0 case_check = _case_sensitive_fs(BUILD_FOLDER) and local_build print("Convert and copy notebook/md files...") for ix_file, page in enumerate(tqdm(list(toc))): url_page = page.get('url', None) title = page.get('title', None) if page.get('external', None): # If its an external link, just pass continue # Make sure URLs (file paths) have correct structure _check_url_page(url_page, CONTENT_FOLDER_NAME) ############################################## # Create path to old/new file and create directory # URL will be relative to the CONTENT_FOLDER path_url_page = os.path.join(PATH_CONTENT_FOLDER, url_page.lstrip('/')) path_url_folder = os.path.dirname(path_url_page) # URLs shouldn't have the suffix in there already so # now we find which one to add for suff in SUPPORTED_FILE_SUFFIXES: if op.exists(path_url_page + suff): path_url_page = path_url_page + suff break elif suff == "#BREAK#": # Final suffix means we didn't find any existing content raise _error( "Could not find file called {} with any of these extensions: {}" .format(path_url_page, SUPPORTED_FILE_SUFFIXES[:-1])) # Create and check new folder / file paths path_build_new_folder = path_url_folder.replace( os.sep + CONTENT_FOLDER_NAME, os.sep + BUILD_FOLDER_NAME) + os.sep path_build_new_file = op.join( path_build_new_folder, op.basename(path_url_page).replace(suff, '.html')) # If the new build file exists and is *newer* than the original file, assume # the original content file hasn't changed and skip it. if overwrite is False and op.exists(path_build_new_file) \ and _file_newer_than(path_build_new_file, path_url_page): n_skipped_files += 1 continue if not op.isdir(path_build_new_folder): os.makedirs(path_build_new_folder) ################################################ # Generate previous/next page URLs if ix_file == 0: url_prev_page = '' prev_file_title = '' else: prev_file_title = toc[ix_file - 1].get('title') url_prev_page = toc[ix_file - 1].get('url') pre_external = toc[ix_file - 1].get('external', False) if pre_external is False: url_prev_page = _prepare_url(url_prev_page) if ix_file == len(toc) - 1: url_next_page = '' next_file_title = '' else: next_file_title = toc[ix_file + 1].get('title') url_next_page = toc[ix_file + 1].get('url') next_external = toc[ix_file + 1].get('external', False) if next_external is False: url_next_page = _prepare_url(url_next_page) ############################################################################### # Get kernel name and presence of widgets from notebooks metadata kernel_name = '' data = jpt.read(path_url_page) if 'metadata' in data and 'kernelspec' in data['metadata']: kernel_name = data['metadata']['kernelspec']['name'] has_widgets = "true" if any( "interactive" in cell['metadata'].get('tags', []) for cell in data['cells']) else "false" ############################################ # Content conversion # Convert notebooks or just copy md if no notebook. if any(path_url_page.endswith(ii) for ii in ['.md', '.ipynb']): # Decide the path where the images will be placed, relative to the HTML location path_after_build_folder = path_build_new_folder.split( os.sep + BUILD_FOLDER_NAME + os.sep)[-1] path_images_new_folder = op.join(PATH_IMAGES_FOLDER, path_after_build_folder) # Build the HTML for this book build_page(path_url_page, path_build_new_folder, path_images_new_folder, path_template=path_template, kernel_name=kernel_name, execute=execute) else: raise _error("Files must end in ipynb or md. Found file {}".format( path_url_page)) ############################################################################### # Modify the generated HTML to work with the SSG with open(path_build_new_file, 'r', encoding='utf8') as ff: lines = ff.readlines() # Split off original yaml yaml_orig, lines = _split_yaml(lines) # Front-matter YAML yaml_fm = [] yaml_fm += ['---'] # In case pre-existing links are sanitized sanitized = url_page.lower().replace('_', '-') if sanitized != url_page: if case_check and url_page.lower() == sanitized: raise RuntimeError( 'Redirect {} clashes with page {} for local build on ' 'case-insensitive FS\n'.format(sanitized, url_page) + 'Rename source page to lower case or build on a case ' 'sensitive FS, e.g. case-sensitive disk image on Mac') yaml_fm += ['redirect_from:'] yaml_fm += [' - "{}"'.format(sanitized)] # Add interactive kernel info interact_path = CONTENT_FOLDER_NAME + '/' + \ path_url_page.split(CONTENT_FOLDER_NAME + '/')[-1] yaml_fm += ['interact_link: {}'.format(interact_path)] yaml_fm += ["kernel_name: {}".format(kernel_name)] yaml_fm += ["has_widgets: {}".format(has_widgets)] # Page metadata # Use YAML block scalars for titles so that people can use special characters # See http://blogs.perl.org/users/tinita/2018/03/strings-in-yaml---to-quote-or-not-to-quote.html yaml_fm += ["title: |-"] yaml_fm += [" {}".format(title)] yaml_fm += ['prev_page:'] yaml_fm += [' url: {}'.format(url_prev_page)] yaml_fm += [" title: |-"] yaml_fm += [" {}".format(prev_file_title)] yaml_fm += ['next_page:'] yaml_fm += [' url: {}'.format(url_next_page)] yaml_fm += [" title: |-"] yaml_fm += [" {}".format(next_file_title)] # Add back any original YaML, and end markers yaml_fm += yaml_orig yaml_fm += [ 'comment: "***PROGRAMMATICALLY GENERATED, DO NOT EDIT. SEE ORIGINAL FILES IN /{}***"' .format(CONTENT_FOLDER_NAME) ] yaml_fm += ['---'] yaml_fm = [ii + '\n' for ii in yaml_fm] lines = yaml_fm + lines # Write the result as UTF-8. with open(path_build_new_file, 'w', encoding='utf8') as ff: ff.writelines(lines) n_built_files += 1 ####################################################### # Finishing up... # Copy non-markdown files in notebooks/ in case they're referenced in the notebooks print('Copying non-content files inside `{}/`...'.format( CONTENT_FOLDER_NAME)) _copy_non_content_files(PATH_CONTENT_FOLDER, CONTENT_FOLDER_NAME, BUILD_FOLDER_NAME) # Message at the end msg = [ "Generated {} new files\nSkipped {} already-built files".format( n_built_files, n_skipped_files) ] if n_built_files == 0: msg += [ "Delete the markdown/HTML files in '{}' for any pages that you wish to re-build, or use --overwrite option to re-build all." .format(BUILD_FOLDER_NAME) ] msg += ["Your Jupyter Book is now in `{}/`.".format(BUILD_FOLDER_NAME)] msg += ["Demo your Jupyter book with `make serve` or push to GitHub!"] print_message_box('\n'.join(msg))