def test_sync_pandoc(nb_file, tmpdir, capsys): tmp_ipynb = str(tmpdir.join('notebook.ipynb')) tmp_md = str(tmpdir.join('notebook.md')) nb = read(nb_file) write(nb, tmp_ipynb) # Test that sync issues a warning when the notebook is not paired jupytext(['--sync', tmp_ipynb]) _, err = capsys.readouterr() assert 'is not a paired notebook' in err # Now with a pairing information nb.metadata.setdefault('jupytext', {})['formats'] = 'ipynb,md:pandoc' write(nb, tmp_ipynb) # Test that missing files are created jupytext(['--sync', tmp_ipynb]) assert os.path.isfile(tmp_md) compare_notebooks(nb, read(tmp_md), 'md:pandoc') with open(tmp_md) as fp: assert 'pandoc' in fp.read()
def test_sync_pandoc(nb_file, tmpdir, capsys): tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_md = str(tmpdir.join("notebook.md")) nb = read(nb_file) write(nb, tmp_ipynb) # Test that sync issues a warning when the notebook is not paired jupytext(["--sync", tmp_ipynb]) _, err = capsys.readouterr() assert "is not a paired notebook" in err # Now with a pairing information nb.metadata.setdefault("jupytext", {})["formats"] = "ipynb,md:pandoc" write(nb, tmp_ipynb) # Test that missing files are created jupytext(["--sync", tmp_ipynb]) assert os.path.isfile(tmp_md) compare_notebooks(read(tmp_md), nb, "md:pandoc") with open(tmp_md) as fp: assert "pandoc" in fp.read()
def render_rmd(input_file: str, output_file: str, params: dict = None): """ Wrapper function to render an Rmarkdown document with the R `rmarkdown` package and convert it to HTML using pandoc and a custom template. Parameters ---------- input_file path to input (Rmd) file output_file path to output (html) file params Dictionary that will be passed to `params` arg of `rmarkdown::render`. See https://bookdown.org/yihui/rmarkdown/parameterized-reports.html for more details. """ # Directory the notebook is located in. Will be used as additional resource path for pandoc. nb_dir = os.path.abspath(os.path.dirname(input_file)) with TemporaryDirectory() as tmp_dir: with TemporaryDirectory() as tmp_dir_nb_converted: # Create this file manually with given name # so that pandoc gracefully falls back to the # file name if no title is specified within the file. (#17) tmp_nb_converted = os.path.join( tmp_dir_nb_converted, os.path.splitext(os.path.basename(input_file))[0] + ".Rmd", ) if not input_file.endswith(".Rmd"): nb = jtx.read(input_file) jtx.write(nb, tmp_nb_converted) else: copyfile(input_file, tmp_nb_converted) md_file = _run_rmarkdown(tmp_nb_converted, tmp_dir, params) run_pandoc(md_file, output_file, res_path=[nb_dir, tmp_dir])
def test_pre_commit_hook_ipynb_to_py( tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev ): """Here we document and test the expected behavior of the pre-commit hook in the directional (--to) mode. Note that here, the ipynb file is always the source for updates - i.e. changes on the .py file will not trigger the hook. """ # set up the tmpdir repo with pre-commit pre_commit_config_yaml = f""" repos: - repo: {jupytext_repo_root} rev: {jupytext_repo_rev} hooks: - id: jupytext args: [--from, ipynb, --to, "py:percent"] """ tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks"]) # write test notebook and output file nb = new_notebook(cells=[new_markdown_cell("A short notebook")]) write(nb, "test.ipynb") jupytext(["--from", "ipynb", "--to", "py:percent", "test.ipynb"]) tmp_repo.git.add(".") tmp_repo.index.commit("test") # make a change to the notebook nb = new_notebook(cells=[new_markdown_cell("Some other text")]) write(nb, "test.ipynb") tmp_repo.git.add("test.ipynb") # now a commit will fail, and keep failing until we add the new # changes made to the existing output to the index ourselves with pytest.raises(HookExecutionError, match="files were modified by this hook"): tmp_repo.index.commit("fails") with pytest.raises(HookExecutionError, match="git add test.py"): tmp_repo.index.commit("fails again") # once we add the changes, it will pass tmp_repo.git.add("test.py") tmp_repo.index.commit("succeeds") assert "test.ipynb" in tmp_repo.tree() assert "test.py" in tmp_repo.tree() # Updating the .py file is possible nb = new_notebook(cells=[new_markdown_cell("Some updated text")]) write(nb, "test.py", fmt="py:percent") tmp_repo.index.commit("update py version") # But it won't change the ipynb file (if you want that, use the --sync mode) nb = read("test.ipynb") assert nb.cells == [new_markdown_cell("Some other text")]
def test_sync_pipe_config(tmpdir): """Sync a notebook to a script paired in a tree, and reformat the markdown cells using pandoc""" tmpdir.join("jupytext.toml").write( """# By default, the notebooks in this repository are in the notebooks subfolder # and they are paired to scripts in the script subfolder. default_jupytext_formats = "notebooks///ipynb,scripts///py:percent" """ ) nb_file = tmpdir.mkdir("notebooks").join("wrap_markdown.ipynb") long_text = "This is a " + ("very " * 24) + "long sentence." assert len(long_text) > 100 nb = new_notebook(cells=[new_markdown_cell(long_text)]) write(nb, str(nb_file)) jupytext( [ "--sync", "--pipe-fmt", "ipynb", "--pipe", "pandoc --from ipynb --to ipynb --atx-headers", str(nb_file), ] ) py_text = tmpdir.join("scripts").join("wrap_markdown.py").read() assert "This is a very very" in py_text for line in py_text.splitlines(): assert len(line) <= 79 nb = read(nb_file, as_version=4) text = nb.cells[0].source assert len(text.splitlines()) == 3 assert text != long_text
def test_pre_commit_hook(tmpdir): tmp_ipynb = str(tmpdir.join("nb with spaces.ipynb")) tmp_py = str(tmpdir.join("nb with spaces.py")) nb = new_notebook(cells=[]) git = git_in_tmpdir(tmpdir) hook = str(tmpdir.join(".git/hooks/pre-commit")) with open(hook, "w") as fp: fp.write("#!/bin/sh\n" "jupytext --to py:light --pre-commit\n") st = os.stat(hook) os.chmod(hook, st.st_mode | stat.S_IEXEC) write(nb, tmp_ipynb) assert os.path.isfile(tmp_ipynb) assert not os.path.isfile(tmp_py) git("add", "nb with spaces.ipynb") git("status") git("commit", "-m", "created") git("status") assert "nb with spaces.py" in git("ls-tree", "-r", "master", "--name-only") assert os.path.isfile(tmp_py)
def test_combine_same_version_ok(tmpdir): tmp_ipynb = str(tmpdir.join('notebook.ipynb')) tmp_nbpy = str(tmpdir.join('notebook.py')) tmp_rmd = str(tmpdir.join('notebook.Rmd')) with open(tmp_nbpy, 'w') as fp: fp.write("""# --- # jupyter: # jupytext_formats: ipynb,py # jupytext_format_version: '1.2' # --- # New cell """) nb = new_notebook(metadata={'jupytext_formats': 'ipynb,py'}) write(nb, tmp_ipynb) # to jupyter notebook jupytext([tmp_nbpy, '--to', 'ipynb', '--update']) # test round trip jupytext([tmp_nbpy, '--to', 'notebook', '--test']) # test ipynb to rmd jupytext([tmp_ipynb, '--to', 'rmarkdown']) nb = read(tmp_ipynb) cells = nb['cells'] assert len(cells) == 1 assert cells[0].cell_type == 'markdown' assert cells[0].source == 'New cell' nb = read(tmp_rmd) cells = nb['cells'] assert len(cells) == 1 assert cells[0].cell_type == 'markdown' assert cells[0].source == 'New cell'
def test_combine_lower_version_raises(tmpdir): tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_nbpy = str(tmpdir.join("notebook.py")) with open(tmp_nbpy, "w") as fp: fp.write("""# --- # jupyter: # jupytext: # formats: ipynb,py:light # text_representation: # extension: .py # format_name: light # format_version: '55.4' # jupytext_version: 42.1.1 # kernelspec: # display_name: Python 3 # language: python # name: python3 # --- # New cell """) nb = new_notebook(metadata={"jupytext_formats": "ipynb,py"}) write(nb, tmp_ipynb) with pytest.raises( ValueError, match= "The file notebook.py was generated with jupytext version 42.1.1 but you have .* installed. " "Please upgrade jupytext to version 42.1.1, or remove either notebook.py or notebook.ipynb. " "This error occurs because notebook.py is in the light format in version 55.4, " "while jupytext version .* installed at .* can only read the light format in versions .*", ): jupytext([tmp_nbpy, "--to", "ipynb", "--update"])
def test_combine_same_version_ok(tmpdir): tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_nbpy = str(tmpdir.join("notebook.py")) tmp_rmd = str(tmpdir.join("notebook.Rmd")) with open(tmp_nbpy, "w") as fp: fp.write("""# --- # jupyter: # jupytext_formats: ipynb,py # jupytext_format_version: '1.2' # --- # New cell """) nb = new_notebook(metadata={"jupytext_formats": "ipynb,py"}) write(nb, tmp_ipynb) # to jupyter notebook jupytext([tmp_nbpy, "--to", "ipynb", "--update"]) # test round trip jupytext([tmp_nbpy, "--to", "notebook", "--test"]) # test ipynb to rmd jupytext([tmp_ipynb, "--to", "rmarkdown"]) nb = read(tmp_ipynb) cells = nb["cells"] assert len(cells) == 1 assert cells[0].cell_type == "markdown" assert cells[0].source == "New cell" nb = read(tmp_rmd) cells = nb["cells"] assert len(cells) == 1 assert cells[0].cell_type == "markdown" assert cells[0].source == "New cell"
def test_manual_call_of_pre_commit_hook(tmpdir): tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_py = str(tmpdir.join("notebook.py")) nb = new_notebook(cells=[]) os.chdir(str(tmpdir)) git = git_in_tmpdir(tmpdir) def hook(): with mock.patch("jupytext.cli.system", system_in_tmpdir(tmpdir)): jupytext(["--to", "py", "--pre-commit"]) write(nb, tmp_ipynb) assert os.path.isfile(tmp_ipynb) assert not os.path.isfile(tmp_py) git("add", "notebook.ipynb") git("status") hook() git("commit", "-m", "created") git("status") assert "notebook.py" in git("ls-tree", "-r", "master", "--name-only") assert os.path.isfile(tmp_py)
def test_rst2md(tmpdir): tmp_py = str(tmpdir.join('notebook.py')) tmp_ipynb = str(tmpdir.join('notebook.ipynb')) # Write notebook in sphinx format nb = new_notebook(cells=[ new_markdown_cell('A short sphinx notebook'), new_markdown_cell(':math:`1+1`') ]) write(nb, tmp_py, fmt='py:sphinx') jupytext([ tmp_py, '--from', 'py:sphinx', '--to', 'ipynb', '--opt', 'rst2md=True', '--opt', 'cell_metadata_filter=-all' ]) assert os.path.isfile(tmp_ipynb) nb = read(tmp_ipynb) assert nb.metadata['jupytext']['cell_metadata_filter'] == '-all' assert nb.metadata['jupytext']['rst2md'] is False # Was rst to md conversion effective? assert nb.cells[2].source == '$1+1$'
def test_apply_black_through_jupytext(tmpdir, nb_file): # Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work metadata = read(nb_file).metadata nb_org = new_notebook(cells=[new_code_cell("1 +1")], metadata=metadata) nb_black = new_notebook(cells=[new_code_cell("1 + 1")], metadata=metadata) os.makedirs(str(tmpdir.join("notebook_folder"))) os.makedirs(str(tmpdir.join("script_folder"))) tmp_ipynb = str(tmpdir.join("notebook_folder").join("notebook.ipynb")) tmp_py = str(tmpdir.join("script_folder").join("notebook.py")) # Black in place write(nb_org, tmp_ipynb) jupytext([tmp_ipynb, "--pipe", "black"]) nb_now = read(tmp_ipynb) compare(nb_now, nb_black) # Write to another folder using dots script_fmt = os.path.join("..", "script_folder//py:percent") write(nb_org, tmp_ipynb) jupytext([tmp_ipynb, "--to", script_fmt, "--pipe", "black"]) assert os.path.isfile(tmp_py) nb_now = read(tmp_py) nb_now.metadata = metadata compare(nb_now, nb_black) os.remove(tmp_py) # Map to another folder based on file name write(nb_org, tmp_ipynb) jupytext( [ tmp_ipynb, "--from", "notebook_folder//ipynb", "--to", "script_folder//py:percent", "--pipe", "black", "--check", "flake8", ] ) assert os.path.isfile(tmp_py) nb_now = read(tmp_py) nb_now.metadata = metadata compare(nb_now, nb_black)
def test_format_prefix_suffix(tmpdir): os.makedirs(str(tmpdir.join("notebooks"))) tmp_ipynb = str(tmpdir.join("notebooks/notebook_name.ipynb")) tmp_py = str(tmpdir.join("scripts/notebook_name.py")) write(new_notebook(), tmp_ipynb) jupytext([tmp_ipynb, "--to", os.path.join("..", "scripts//py")]) assert os.path.isfile(tmp_py) os.remove(tmp_py) jupytext([tmp_ipynb, "--to", "scripts//py", "--from", "notebooks//ipynb"]) assert os.path.isfile(tmp_py) os.remove(tmp_py) tmp_ipynb = str(tmpdir.join("notebooks/nb_prefix_notebook_name.ipynb")) tmp_py = str(tmpdir.join("scripts/script_prefix_notebook_name.py")) write(new_notebook(), tmp_ipynb) jupytext( [ tmp_ipynb, "--to", "scripts/script_prefix_/py", "--from", "notebooks/nb_prefix_/ipynb", ] ) assert os.path.isfile(tmp_py) os.remove(tmp_py) tmp_ipynb = str(tmpdir.join("notebooks/nb_prefix_notebook_name_nb_suffix.ipynb")) tmp_py = str(tmpdir.join("scripts/script_prefix_notebook_name_script_suffix.py")) write(new_notebook(), tmp_ipynb) jupytext( [ tmp_ipynb, "--to", "scripts/script_prefix_/_script_suffix.py", "--from", "notebooks/nb_prefix_/_nb_suffix.ipynb", ] ) assert os.path.isfile(tmp_py) os.remove(tmp_py)
def test_apply_black_through_jupytext(tmpdir, nb_file): # Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work metadata = read(nb_file).metadata nb_org = new_notebook(cells=[new_code_cell('1 +1')], metadata=metadata) nb_black = new_notebook(cells=[new_code_cell('1 + 1')], metadata=metadata) os.makedirs(str(tmpdir.join('notebook_folder'))) os.makedirs(str(tmpdir.join('script_folder'))) tmp_ipynb = str(tmpdir.join('notebook_folder').join('notebook.ipynb')) tmp_py = str(tmpdir.join('script_folder').join('notebook.py')) # Black in place write(nb_org, tmp_ipynb) jupytext([tmp_ipynb, '--pipe', 'black']) nb_now = read(tmp_ipynb) compare(nb_now, nb_black) # Write to another folder using dots script_fmt = os.path.join('..', 'script_folder//py:percent') write(nb_org, tmp_ipynb) jupytext([tmp_ipynb, '--to', script_fmt, '--pipe', 'black']) assert os.path.isfile(tmp_py) nb_now = read(tmp_py) nb_now.metadata = metadata compare(nb_now, nb_black) os.remove(tmp_py) # Map to another folder based on file name write(nb_org, tmp_ipynb) jupytext([ tmp_ipynb, '--from', 'notebook_folder//ipynb', '--to', 'script_folder//py:percent', '--pipe', 'black', '--check', 'flake8' ]) assert os.path.isfile(tmp_py) nb_now = read(tmp_py) nb_now.metadata = metadata compare(nb_now, nb_black)
def test_format_prefix_suffix(tmpdir): os.makedirs(str(tmpdir.join('notebooks'))) tmp_ipynb = str(tmpdir.join('notebooks/notebook_name.ipynb')) tmp_py = str(tmpdir.join('scripts/notebook_name.py')) write(new_notebook(), tmp_ipynb) jupytext([tmp_ipynb, '--to', os.path.join('..', 'scripts//py')]) assert os.path.isfile(tmp_py) os.remove(tmp_py) jupytext([tmp_ipynb, '--to', 'scripts//py', '--from', 'notebooks//ipynb']) assert os.path.isfile(tmp_py) os.remove(tmp_py) tmp_ipynb = str(tmpdir.join('notebooks/nb_prefix_notebook_name.ipynb')) tmp_py = str(tmpdir.join('scripts/script_prefix_notebook_name.py')) write(new_notebook(), tmp_ipynb) jupytext([ tmp_ipynb, '--to', 'scripts/script_prefix_/py', '--from', 'notebooks/nb_prefix_/ipynb' ]) assert os.path.isfile(tmp_py) os.remove(tmp_py) tmp_ipynb = str( tmpdir.join('notebooks/nb_prefix_notebook_name_nb_suffix.ipynb')) tmp_py = str( tmpdir.join('scripts/script_prefix_notebook_name_script_suffix.py')) write(new_notebook(), tmp_ipynb) jupytext([ tmp_ipynb, '--to', 'scripts/script_prefix_/_script_suffix.py', '--from', 'notebooks/nb_prefix_/_nb_suffix.ipynb' ]) assert os.path.isfile(tmp_py) os.remove(tmp_py)
def test_pre_commit_hook_sync_reformat_code_and_markdown( tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev, notebook_with_outputs, ): """Here we sync the ipynb notebook with a py:percent file and also apply black and pandoc to reformat both code and markdown cells. Note: the new cell ids introduced in nbformat 5.1.0 are not yet supported by all the programs that treat ipynb files. Consequently we pin the version of nbformat to 5.0.8 in all the environments below (and you will have to do the same on the environment in which you edit the notebooks). """ pre_commit_config_yaml = f""" repos: - repo: {jupytext_repo_root} rev: {jupytext_repo_rev} hooks: - id: jupytext args: [--sync, --pipe-fmt, ipynb, --pipe, 'pandoc --from ipynb --to ipynb --markdown-headings=atx', --diff] additional_dependencies: - nbformat==5.0.8 # because pandoc 2.11.4 does not preserve yet the new cell ids - id: jupytext args: [--sync, --pipe, black, --diff] additional_dependencies: - black==20.8b1 # Matches black hook below - nbformat==5.0.8 # for compatibility with the pandoc hook above - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black """ tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) tmpdir.join(".jupytext.toml").write('formats = "ipynb,py:percent"') tmp_repo.git.add(".jupytext.toml") tmp_repo.index.commit("pair notebooks") # write a test notebook notebook = new_notebook( cells=[ new_code_cell("1+1"), new_markdown_cell("""This is a complex markdown cell # With a h1 header ## And a h2 header | And | A | Table | | --------- | ---| ----- | | 0 | 1 | 2 | !!!WARNING!!! This hook does not seem compatible with explicit paragraph breaks (two spaces at the end of a line). And a VERY long line. """.replace("VERY ", "very " * 51)), ], metadata=notebook_with_outputs.metadata, ) # We write the notebook in version 4.4, i.e. with the equivalent of nbformat version 5.0.8 notebook.nbformat = 4 notebook.nbformat_minor = 4 write(notebook, "test.ipynb") # try to commit it, should fail because # 1. the py version hasn't been added # 2. the first cell is '1+1' which is not black compliant # 3. the second cell needs to be wrapped tmp_repo.git.add("test.ipynb") with pytest.raises( HookExecutionError, match="files were modified by this hook", ): tmp_repo.index.commit("failing") # Add the two files tmp_repo.git.add("test.ipynb") tmp_repo.git.add("test.py") # now the commit will succeed tmp_repo.index.commit("passing") assert "test.ipynb" in tmp_repo.tree() assert "test.py" in tmp_repo.tree() # both the code and the markdown cells were reformatted nb = read("test.ipynb") assert nb.cells[0].source == "1 + 1" print(nb.cells[1].source) assert (nb.cells[1].source == """This is a complex markdown cell # With a h1 header ## And a h2 header | And | A | Table | |-----|-----|-------| | 0 | 1 | 2 | !!!WARNING!!! This hook does not seem compatible with explicit paragraph breaks (two spaces at the end of a line). And a very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very very long line.""")
def test_write_non_ascii(tmpdir): nb = jupytext.reads(u"Non-ascii contênt", "Rmd") jupytext.write(nb, str(tmpdir.join("notebook.Rmd"))) jupytext.write(nb, str(tmpdir.join("notebook.ipynb")))
def create_mirror_file_if_missing(mirror_file, notebook, fmt): if not os.path.isfile(mirror_file): jupytext.write(notebook, mirror_file, fmt=fmt)
def tmp_ipynb(tmpdir): tmp_file = str(tmpdir.join("notebook.ipynb")) write(new_notebook(), tmp_file) return tmp_file
def test_pre_commit_hook_sync_black_nbstripout( tmpdir, cwd_tmpdir, tmp_repo, jupytext_repo_root, jupytext_repo_rev, notebook_with_outputs, ): """Here we sync the ipynb notebook with a py:percent file and also apply black and nbstripout.""" pre_commit_config_yaml = f""" repos: - repo: {jupytext_repo_root} rev: {jupytext_repo_rev} hooks: - id: jupytext args: [--sync, --pipe, black] additional_dependencies: - black==20.8b1 # Matches hook - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black - repo: https://github.com/kynan/nbstripout rev: 0.3.9 hooks: - id: nbstripout """ tmpdir.join(".pre-commit-config.yaml").write(pre_commit_config_yaml) tmp_repo.git.add(".pre-commit-config.yaml") pre_commit(["install", "--install-hooks", "-f"]) tmpdir.join(".jupytext.toml").write('default_jupytext_formats = "ipynb,py:percent"') tmp_repo.git.add(".jupytext.toml") tmp_repo.index.commit("pair notebooks") # write a test notebook write(notebook_with_outputs, "test.ipynb") # try to commit it, should fail because # 1. the py version hasn't been added # 2. the first cell is '1+1' which is not black compliant # 3. the notebook has outputs tmp_repo.git.add("test.ipynb") with pytest.raises( HookExecutionError, match="files were modified by this hook", ): tmp_repo.index.commit("failing") # Add the two files tmp_repo.git.add("test.ipynb") tmp_repo.git.add("test.py") # now the commit will succeed tmp_repo.index.commit("passing") assert "test.ipynb" in tmp_repo.tree() assert "test.py" in tmp_repo.tree() # the first cell was reformatted nb = read("test.ipynb") assert nb.cells[0].source == "1 + 1" # the ipynb file has no outputs assert not nb.cells[0].outputs
def test_meaningfull_error_write_myst_missing(tmpdir): nb_file = tmpdir.join("notebook.ipynb") jupytext.write(new_notebook(), str(nb_file)) with pytest.raises(ImportError, match=PLEASE_INSTALL_MYST): jupytext_cli([str(nb_file), "--to", "md:myst"])
def test_write_wrong_ext(tmpdir, nb_file='notebook.ext'): nb_file = str(tmpdir.join(nb_file)) with pytest.raises(JupytextFormatError): jupytext.write(new_notebook(), nb_file)
def develop(self, app='notebook', args=None): """ Opens the rendered notebook (with injected parameters) and adds a "debugging-settings" cell to the that changes directory to the current active directory. This will reflect conditions when callign `DAG.build()`. This modified notebook is saved in the same location as the source with a "-tmp" added to the filename. Changes to this notebook can be exported to the original notebook after the notebook process is shut down. The "injected-parameters" and "debugging-settings" cells are deleted before saving. Parameters ---------- app : {'notebook', 'lab'}, default: 'notebook' Which Jupyter application to use args : str Extra parameters passed to the jupyter application Notes ----- Be careful when developing tasks interacively. If the task has run successfully, you overwrite products but don't save the updated source code, your DAG will enter an inconsistent state where the metadata won't match the overwritten product. If you modify the source code and call develop again, the source code will be updated only if the ``hot_reload option`` is turned on. See :class:`ploomber.DAGConfigurator` for details. """ # TODO: this code needs refactoring, should be a context manager # like the one we have for PythonCallable.develop that abstracts # the handling of the temporary notebook while editing apps = {'notebook', 'lab'} if app not in apps: raise ValueError('"app" must be one of {}, got: "{}"'.format( apps, app)) if self.source.language != 'python': raise NotImplementedError( 'develop is not implemented for "{}" ' 'notebooks, only python is supported'.format( self.source.language)) if self.source.loc is None: raise ValueError('Can only use develop in notebooks loaded ' 'from files, not from str') nb = _read_rendered_notebook(self.source.nb_str_rendered) name = self.source.loc.name suffix = self.source.loc.suffix name_new = name.replace(suffix, '-tmp.ipynb') tmp = self.source.loc.with_name(name_new) content = nbformat.writes(nb, version=nbformat.NO_CONVERT) tmp.write_text(content) # open notebook with injected debugging cell try: subprocess.run(['jupyter', app, str(tmp)] + shlex.split(args or ''), check=True) except KeyboardInterrupt: print(f'Jupyter {app} application closed...') # read tmp file again, to see if the user made any changes content_new = Path(tmp).read_text() # maybe exclude changes in tmp cells? if content == content_new: print('No changes found...') else: # save changes if _save(): nb = nbformat.reads(content_new, as_version=nbformat.NO_CONVERT) # remove injected-parameters and debugging-settings cells if # they exist _cleanup_rendered_nb(nb) # write back in the same format and original location ext_source = Path(self.source.loc).suffix[1:] print('Saving notebook to: ', self.source.loc) jupytext.write(nb, self.source.loc, fmt=ext_source) else: print('Not saving changes...') # remove tmp file Path(tmp).unlink()
def convert_to_myst(file, dest_path): # Read a notebook from a file ntbk = jupytext.read(file, fmt='ipynb') # Write notebook to md file jupytext.write(ntbk, dest_path.joinpath(file.stem + '.md'), fmt='md')
def test_sync_with_pre_commit_hook(tmpdir): # Init git and create a pre-commit hook git = git_in_tmpdir(tmpdir) git('init') git('status') hook = str(tmpdir.join('.git/hooks/pre-commit')) with open(hook, 'w') as fp: fp.write('#!/bin/sh\n' 'jupytext --sync --pre-commit\n') st = os.stat(hook) os.chmod(hook, st.st_mode | stat.S_IEXEC) # Create a notebook that is not paired tmp_ipynb = str(tmpdir.join('notebook.ipynb')) tmp_md = str(tmpdir.join('notebook.md')) nb = new_notebook(cells=[new_markdown_cell('A short notebook')]) write(nb, tmp_ipynb) assert os.path.isfile(tmp_ipynb) assert not os.path.isfile(tmp_md) git('add', 'notebook.ipynb') git('status') git('commit', '-m', 'created') git('status') assert 'notebook.ipynb' in git('ls-tree', '-r', 'master', '--name-only') assert 'notebook.md' not in git('ls-tree', '-r', 'master', '--name-only') assert os.path.isfile(tmp_ipynb) assert not os.path.exists(tmp_md) # Pair the notebook jupytext(['--set-formats', 'ipynb,md', tmp_ipynb]) # Remove the md file (it will be regenerated by the pre-commit hook) os.remove(tmp_md) # Commit the ipynb file git('add', 'notebook.ipynb') git('status') git('commit', '-m', 'paired') git('status') # The pre-commit script should have created and committed the md file assert 'notebook.ipynb' in git('ls-tree', '-r', 'master', '--name-only') assert 'notebook.md' in git('ls-tree', '-r', 'master', '--name-only') assert os.path.isfile(tmp_md) nb_md = read(tmp_md) compare_notebooks(nb_md, nb) # Edit the md file with open(tmp_md) as fp: md_text = fp.read() with open(tmp_md, 'w') as fp: fp.write(md_text.replace('A short notebook', 'Notebook was edited')) # commit the md file git('add', 'notebook.md') git('status') git('commit', '-m', 'edited md') git('status') # The pre-commit script should have sync and committed the ipynb file assert 'notebook.ipynb' in git('ls-tree', '-r', 'master', '--name-only') assert 'notebook.md' in git('ls-tree', '-r', 'master', '--name-only') nb = read(tmp_ipynb) compare(nb.cells, [new_markdown_cell('Notebook was edited')])
import os import shutil import jupytext from tools import send_feedback submission_dir = "/shared/submission/" # submission_dir = "sub2/" for file in os.listdir(submission_dir): if file.endswith('.ipynb'): learner_notebook = file else: learner_notebook = None if learner_notebook is None: send_feedback(0.0, "No notebook was found in the submission directory.") exit() sub_source = submission_dir + learner_notebook sub_destination = '/grader/submission/submission.ipynb' # sub_destination = 'submission/submission.ipynb' shutil.copyfile(sub_source, sub_destination) nb = jupytext.read("submission/submission.ipynb") jupytext.write(nb, 'submission/submission.py', fmt='py:percent')
path_loader=data_path, path_rf=data_path, path_gabor=data_path, ) path_output.mkdir(parents=True, exist_ok=True) for nb in nbs: try: ntb = jupytext.read(nb + ".py") ntb.metadata["kernelspec"] = { "display_name": "Python 3", "language": "python", "name": "python3", } jupytext.write(ntb, nb + ".ipynb") print(nb) pm.execute_notebook( f"{nb}.ipynb", output_path=(path_output / f"{nb}_output.ipynb").as_posix(), parameters=parameters, ) except pm.exceptions.PapermillExecutionError as e: logging.error(f"Error at {nb}.") raise e finally: Path(nb + ".ipynb").unlink(missing_ok=True)
def test_sync_with_pre_commit_hook(tmpdir): # Init git and create a pre-commit hook git = git_in_tmpdir(tmpdir) hook = str(tmpdir.join(".git/hooks/pre-commit")) with open(hook, "w") as fp: fp.write("#!/bin/sh\n" "jupytext --sync --pre-commit\n") st = os.stat(hook) os.chmod(hook, st.st_mode | stat.S_IEXEC) # Create a notebook that is not paired tmp_ipynb = str(tmpdir.join("notebook.ipynb")) tmp_md = str(tmpdir.join("notebook.md")) nb = new_notebook(cells=[new_markdown_cell("A short notebook")]) write(nb, tmp_ipynb) assert os.path.isfile(tmp_ipynb) assert not os.path.isfile(tmp_md) git("add", "notebook.ipynb") git("status") git("commit", "-m", "created") git("status") assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only") assert "notebook.md" not in git("ls-tree", "-r", "master", "--name-only") assert os.path.isfile(tmp_ipynb) assert not os.path.exists(tmp_md) # Pair the notebook jupytext(["--set-formats", "ipynb,md", tmp_ipynb]) # Remove the md file (it will be regenerated by the pre-commit hook) os.remove(tmp_md) # Commit the ipynb file git("add", "notebook.ipynb") git("status") git("commit", "-m", "paired") git("status") # The pre-commit script should have created and committed the md file assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only") assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only") assert os.path.isfile(tmp_md) nb_md = read(tmp_md) compare_notebooks(nb_md, nb) # Edit the md file with open(tmp_md) as fp: md_text = fp.read() with open(tmp_md, "w") as fp: fp.write(md_text.replace("A short notebook", "Notebook was edited")) # commit the md file git("add", "notebook.md") git("status") git("commit", "-m", "edited md") git("status") # The pre-commit script should have sync and committed the ipynb file assert "notebook.ipynb" in git("ls-tree", "-r", "master", "--name-only") assert "notebook.md" in git("ls-tree", "-r", "master", "--name-only") nb = read(tmp_ipynb) compare(nb.cells, [new_markdown_cell("Notebook was edited")]) # create and commit a jpg file tmp_jpg = str(tmpdir.join("image.jpg")) with open(tmp_jpg, "wb") as fp: fp.write(b"") git("add", "image.jpg") git("commit", "-m", "added image")
def to_notebook(path): notebook = jupytext.read(path) jupytext.write(notebook, replace_extension(path, ".ipynb"))
k2: when_first_appear(argsasadded, k2) for k2 in parsingparser_ARGS.keys() } # add args line-by-line to the python file we are building in-memory for k2 in sorted(list(keys2whenapp.keys()), key=lambda x2: keys2whenapp[x2]): assert ' ' not in k2 and '\t' not in k2 and '\n' not in k2 and '\r' not in k2, k2 dumpval = json.dumps( parsingparser_ARGS[k2] ) # TODO: how to properly reproduce python CODE declaration of variable if isinstance(parsingparser_ARGS[k2], bool): dumpval = str(parsingparser_ARGS[k2]) FINALPRINTED += "args." + k2 + " = " + dumpval + "\n" no_category_match = False # if no argparse line category match, just print the line # although, one last conversion fix: replace __file__ with the file path, since jupyter doesnt know __file__ if no_category_match: line = join_tokens(line) line = line.replace( '(__file__)', '(' + json.dumps(os.path.abspath(args.infile)) + ')') if args.eval_until_argparse and parsingparser_ARGS is None: evalline = line.strip('\r\n\t ') if len(evalline) > 0 and evalline[0] != '#': exec(evalline) FINALPRINTED += line + "\n" jupytext.write(jupytext.reads(FINALPRINTED, fmt='py'), args.out_dest)