def convert_notebook(ramp_kit_dir='.'): import nbformat import nbconvert from nbconvert.exporters import HTMLExporter problem_name = os.path.basename(os.path.abspath(ramp_kit_dir)) print('Testing if the notebook can be converted to html') notebook_filename = os.path.join( os.path.abspath(ramp_kit_dir), '{}_starting_kit.ipynb'.format(problem_name)) notebook_html_filename = os.path.join( os.path.abspath(ramp_kit_dir), '{}_starting_kit.html'.format(problem_name)) with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) nb_html, _ = nbconvert.export(HTMLExporter, nb) with open( os.path.join(os.path.abspath(ramp_kit_dir), notebook_html_filename), 'wb') as f: f.write(nb_html.encode('utf-8')) delete_line_from_file(notebook_html_filename, '<link rel="stylesheet" href="custom.css">\n')
def notebook_to_md(nbfile, output_dir): import nbformat, nbconvert, os from collections import OrderedDict nb = nbformat.read(nbfile, as_version=nbformat.NO_CONVERT) dir_name, nb_name = os.path.split(nbfile) base_nb_name = os.path.splitext(nb_name)[0] exporter = nbconvert.get_exporter("markdown") resources = { 'output_files_dir': base_nb_name + "_files", "build_dir": output_dir } mdtext, resources = nbconvert.export(exporter, nb=nb, resources=resources) # Handle metadata specific to docs.microsoft.com meta = nb["metadata"].get("ms_docs_meta", {}) docs_meta = OrderedDict() for key in [ 'title', 'description', 'services', 'author', 'manager', 'ms.service', 'ms.technology', 'ms.topic', 'ms.date', 'ms.author' ]: if key in meta: docs_meta[key] = meta[key] if len(docs_meta) > 0: docs_meta_md = '\n'.join(['---'] + [': '.join(kv) for kv in docs_meta.items()] + ['---']) mdtext = docs_meta_md + '\n' + mdtext writer = nbconvert.writers.FilesWriter() writer.write(mdtext, resources, base_nb_name)
def test_markdown_jupytext_nbconvert_is_identity(md_file): """Test that a Markdown file, converted to a notebook, then exported back to Markdown with nbconvert, yields the original file""" with open(md_file) as fp: md_org = fp.read() nb = jupytext.reads(md_org, "md") import nbconvert md_nbconvert, _ = nbconvert.export(nbconvert.MarkdownExporter, nb) # Our expectations md_expected = md_org.splitlines() # #region and #endregion comments are removed md_expected = [ line for line in md_expected if line not in ["<!-- #region -->", "<!-- #endregion -->"] ] # language is not inserted by nbconvert md_expected = [ "```" if line.startswith("```") else line for line in md_expected ] # nbconvert inserts no empty line after the YAML header (which is in a Raw cell) md_expected = "\n".join(md_expected).replace("---\n\n", "---\n") + "\n" # an extra blank line is inserted before code cells md_nbconvert = md_nbconvert.replace("\n\n```", "\n```") jupytext.compare.compare(md_nbconvert, md_expected)
def jup2py(jup_filename, py_filename=None, save=True): # Converting jupyter to py through nbconvertor py_code = nbconvert.export(nbconvert.PythonExporter(), jup_filename) mycode = py_code[0].split("\n") clean_code = [] for index, line in enumerate(mycode): # Removing which are either "\n" or denotes the "Input" Tag for jupyter notebook if line == "\n" or ( line.startswith("# In[")) or line is None or line == "": pass # Commenting the magic functions or the command line elif line.startswith("get_ipython()"): line = "#" + line clean_code.append(line) # Remove the jup2py line itself elif "jup2py" in line: pass else: clean_code.append(line) clean_code = "\n".join(clean_code) # Saving back the .py file if save: save_pyfile(clean_code, py_filename) else: return clean_code
def convert_to_files(names, to_format): target_dir = os.path.join("static-files", to_format) for folder in folders: if not os.path.exists(os.path.join(target_dir, folder)): os.makedirs(os.path.join(target_dir, folder)) for file_name in names: p = nbconvert.export(nbconvert.get_exporter(to_format), file_name) with open(os.path.join(target_dir, file_name[:-6] + p[1]["output_extension"]), 'w') as f: f.write(p[0].encode("utf-8")) print file_name
def _reformat_notebook(script): """convert a notebook to a script and remove jupyter magics Parameters ---------- script : str The path to the Jupyter notebook Returns ------- code : str The notebook as a python script. """ nb = nbformat.read(script, nbformat.NO_CONVERT) code = nbconvert.export(nbconvert.PythonExporter(), nb)[0] new_code = [] for line in code.split('\n'): # Exclude magics - HACK - TODO # Matthew R. Becker 2/27/2017 # So this one is tough. The following magics # # %matplotlib inline # %matplotlib notebook # # are only available in jupyter. # When notebooks are converted to scripts currently (2/17/2017 w/ # version 5.1.1 of nbconvert), these magics are left in the notebook # as normal ipython magics. However, they cannot be executed by # ipython and cause an error. There is an open issue in nbconvert to # exclude these things # (see https://github.com/jupyter/nbconvert/pull/507). Until this # issue is addressed, we have to parse them out by hand. # The block of code here does the following. # 1) finds jupyter magics at any indentation level (they CAN be # indented!) # 2) replaces them with `pass` in python (which does nothing) at the # same indentation level if not (line.strip().startswith("get_ipython().magic(" "'matplotlib") or line.strip().startswith("get_ipython().run_line_magic(" "'matplotlib") or # python 2 - ugh line.strip().startswith("get_ipython().magic(" "u'matplotlib") or line.strip().startswith("get_ipython().run_line_magic(" "u'matplotlib")): new_code.append(line) else: new_code.append(line[:-len(line.lstrip())] + 'pass') return '\n'.join(new_code)
def convert_to_files(names, to_format): target_dir = os.path.join("static-files", to_format) for folder in folders: if not os.path.exists(os.path.join(target_dir, folder)): os.makedirs(os.path.join(target_dir, folder)) for file_name in names: p = nbconvert.export(nbconvert.get_exporter(to_format), file_name) with open( os.path.join(target_dir, file_name[:-6] + p[1]["output_extension"]), 'w') as f: f.write(p[0].encode("utf-8")) print file_name
def notebook_to_md(nbfile, output_dir): import nbformat, nbconvert, os, shutil from collections import OrderedDict from base64 import b64encode nb = nbformat.read(nbfile, as_version=nbformat.NO_CONVERT) dir_name, nb_name = os.path.split(nbfile) base_nb_name = os.path.splitext(nb_name)[0] exporter = nbconvert.get_exporter("markdown") resources = { 'output_files_dir': base_nb_name + "_files", "build_dir": output_dir } mdtext, resources = nbconvert.export(exporter, nb=nb, resources=resources) # Handle metadata specific to docs.microsoft.com meta = nb["metadata"].get("ms_docs_meta", {}) docs_meta = OrderedDict() for key in [ 'title', 'description', 'services', 'author', 'manager', 'ms.service', 'ms.technology', 'ms.topic', 'ms.date', 'ms.author' ]: if key in meta: docs_meta[key] = meta[key] if len(docs_meta) > 0: docs_meta_md = '\n'.join(['---'] + [': '.join(kv) for kv in docs_meta.items()] + ['---']) mdtext = docs_meta_md + '\n' + mdtext # Replace images containing paths to file with base64-encoded inline images for k, v in resources["outputs"].items(): ext = os.path.splitext(k)[-1][1:] if ext in ("png", "jpg", "svg"): b64 = b64encode(v) v2 = "data:{0};base64,{1}".format(ext, b64.decode("ascii")) mdtext = mdtext.replace(k, v2) writer = nbconvert.writers.FilesWriter() writer.write(mdtext, resources, base_nb_name) shutil.rmtree(resources["output_files_dir"], ignore_errors=True) return mdtext, resources
def run_test(project, zone, cluster, new_values): # pylint: disable=too-many-locals # TODO([email protected]): Need to configure the notebook and test to build # using GCB. dirname = os.path.dirname(__file__) if not dirname: logging.info("__file__ doesn't apper to be absolute path.") dirname = os.getcwd() notebook_path = os.path.join(dirname, "TF on GKE.ipynb") logging.info("Reading notebook %s", notebook_path) if not os.path.exists(notebook_path): raise ValueError("%s does not exist" % notebook_path) with open(notebook_path) as hf: node = nbformat.read(hf, nbformat.NO_CONVERT) exporter = nbconvert.PythonExporter() raw, _ = nbconvert.export(exporter, node) credentials = GoogleCredentials.get_application_default() gke = discovery.build("container", "v1", credentials=credentials) lines = raw.splitlines() modified = replace_vars(lines, new_values) modified = strip_appendix(modified) modified = strip_unexecutable(modified) with tempfile.NamedTemporaryFile(suffix="notebook.py", prefix="tmpGke", mode="w", delete=False) as hf: code_path = hf.name hf.write("\n".join(modified)) logging.info("Wrote notebook to: %s", code_path) with open(code_path) as hf: contents = hf.read() logging.info("Notebook contents:\n%s", contents) try: runpy.run_path(code_path) finally: logging.info("Deleting cluster; project=%s, zone=%s, name=%s", project, zone, cluster) util.delete_cluster(gke, cluster, project, zone)
def test_jupytext_markdown_similar_to_nbconvert(nb_file): """Test that the nbconvert export for a notebook matches Jupytext's one""" nb = jupytext.read(nb_file) # Remove cell outputs and metadata for cell in nb.cells: cell.outputs = [] cell.execution_count = None cell.metadata = {} md_jupytext = jupytext.writes(nb, fmt="md") import nbconvert md_nbconvert, _ = nbconvert.export(nbconvert.MarkdownExporter, nb) # our expectations # nbconvert file has no YAML header md_jupytext_lines = md_jupytext.splitlines() _, _, raw_cell, pos = header_to_metadata_and_cell(md_jupytext_lines, "") md_jupytext = "\n".join(md_jupytext_lines[pos:]) + "\n" if raw_cell is not None: md_jupytext = raw_cell.source + "\n\n" + md_jupytext # region comments are not in nbconvert md_jupytext = md_jupytext.replace("<!-- #region -->\n", "").replace("<!-- #endregion -->\n", "") # Jupytext uses HTML comments to keep track of raw cells md_jupytext = (md_jupytext.replace("\n<!-- #raw -->\n", "").replace( "<!-- #raw -->\n", "").replace("\n<!-- #endraw -->\n", "")) # nbconvert file may start with an empty line md_jupytext = md_jupytext.lstrip("\n") md_nbconvert = md_nbconvert.lstrip("\n") # Jupytext may not always have two blank lines between cells like Jupyter nbconvert md_jupytext = md_jupytext.replace("\n\n\n", "\n\n") md_nbconvert = md_nbconvert.replace("\n\n\n", "\n\n") jupytext.compare.compare(md_nbconvert, md_jupytext)
def _from_ipynb(path_to_nb, exporter, nbconvert_export_kwargs): """ Convert ipynb to another format """ path = Path(path_to_nb) nb = nbformat.reads(path.read_text(), as_version=nbformat.NO_CONVERT) content, _ = nbconvert.export(exporter, nb, **nbconvert_export_kwargs) if isinstance(content, str): path.write_text(content) elif isinstance(content, bytes): path.write_bytes(content) else: raise TypeError('nbconvert returned a converted notebook with' 'unknown format, only text and binary objects ' 'are supported') return content
def test_script(self): nb_path = TEST_FILES_PATH + "autograder/submission/fails2and6H.ipynb" nb = nbformat.read(nb_path, as_version=NBFORMAT_VERSION) os.remove(nb_path) try: py, _ = nbconvert.export(nbconvert.PythonExporter, nb) # remove magic commands py = "\n".join(l for l in py.split("\n") if not l.startswith("get_ipython")) with open(TEST_FILES_PATH + "autograder/submission/fails2and6H.py", "w+") as f: f.write(py) run_autograder(self.config['autograder_dir']) with open(TEST_FILES_PATH + "autograder/results/results.json") as f: actual_results = json.load(f) self.assertEqual( actual_results, self.expected_results, f"Actual results did not matched expected:\n{actual_results}") self.deletePaths( [TEST_FILES_PATH + "autograder/submission/fails2and6H.py"]) with open(nb_path, "w+") as f: nbformat.write(nb, f) except: os.chdir(self.cwd) self.deletePaths( [TEST_FILES_PATH + "autograder/submission/fails2and6H.py"]) with open(nb_path, "w+") as f: nbformat.write(nb, f) raise
def prepare_script(tool_arguments, notebook): script_lines = [] script_lines.append('from sys import argv') for index, k in enumerate(tool_arguments): v = tool_arguments[k] if isinstance(v, string_types): line_template = '{argument_name} = argv[{i}]' else: line_template = '{argument_name} = {value_type}(argv[{i}])' script_lines.append( line_template.format(argument_name=k, i=index + 1, value_type=type(v).__name__)) script_lines.extend( ['\n'] + JINJA2_ENVIRONMENT.get_template('python.jinja2').render().splitlines()) notebook = deepcopy(notebook) code_cells = get_code_cells(notebook) code_cells[0]['source'] = '\n'.join(script_lines) script_content = nbconvert.export(PythonExporter, notebook)[0] return script_content
def _from_ipynb(path_to_nb, extension, nbconvert_exporter_name): if nbconvert_exporter_name is not None: exporter = nbconvert.get_exporter(nbconvert_exporter_name) else: try: exporter = nbconvert.get_exporter(extension.replace('.', '')) except ValueError: raise TaskBuildError('Could not determine nbconvert exporter ' 'either specify in the path extension ' 'or pass a valid exporter name in ' 'the NotebookRunner constructor, ' 'valid expoers are: {}'.format( nbconvert.get_export_names())) path = Path(path_to_nb) nb = nbformat.v4.reads(path.read_text()) content, _ = nbconvert.export(exporter, nb, exclude_input=True) path.write_text(content) return content
def run( self, path: str = None, parameters: dict = None, output_format: str = None, exporter_kwargs: dict = None, ) -> str: """ Run a Jupyter notebook and output as HTML, notebook, or other formats. Args: - path (string, optional): path to fetch the notebook from; can also be a cloud storage path - parameters (dict, optional): dictionary of parameters to use for the notebook - output_format (str, optional): Notebook output format, should be a valid nbconvert Exporter name. 'json' is treated as 'notebook'. Valid exporter names: asciidoc, custom, html, latex, markdown, notebook, pdf, python, rst, script, slides, webpdf. (default: notebook) - exporter_kwargs (dict, optional): The arguments used for initializing the exporter. """ nb: nbformat.NotebookNode = pm.execute_notebook( path, "-", parameters=parameters, kernel_name=self.kernel_name, log_output=self.log_output, ) if output_format == "json": output_format = "notebook" if exporter_kwargs is None: exporter_kwargs = {} exporter = nbconvert.get_exporter(output_format) body, resources = nbconvert.export(exporter, nb, **exporter_kwargs) return body
""" This script is called by sphinxdoc to convert all Jupyter notebooks in the Examples folder to ReStructured Text files and stores them in the documentation folder. Author: R.A. Collenteur 2016 """ import nbconvert from nbconvert import RSTExporter import glob import io import os notebooks = glob.iglob('../examples/notebooks/*.ipynb') for nb in notebooks: x = nbconvert.export(RSTExporter, nb)[0] with io.open('../doc/examples/%s.rst' % os.path.basename(nb)[:-6], 'w', encoding='utf-8') as f: f.write(x[:]) print("example notebooks successfully converted to .rst files")
def run(bytes_in): myzipfile = zipfile.ZipFile(bytes_in) with TemporaryDirectory() as temp_dir: """Automatically deletes temp_dir if an error occurs""" print('\nCreate temporary folder: ', temp_dir) out_formats = ['latex', 'html', 'slides', 'rst', 'markdown'] out_formats_ext = ['tex', 'html', 'slides.html', 'rst', 'md'] # find nam files: all_nam_files = [] for f in myzipfile.infolist(): if f.is_dir(): # folder continue elif f.filename[0] == '_': # private continue elif os.path.basename(f.filename)[0] == '.': # private continue elif f.filename.lower()[-4:] == '.nam': all_nam_files.append(f) else: pass assert len( all_nam_files ) != 0, "The input zip does not contain a .nam nor a .NAM file." assert len( all_nam_files) == 1, "The zip can only contain a single name file." + \ "The nam files in the zip are: {0}".format(all_nam_files) # directory name nam file nam_dir = os.path.join(os.path.dirname(all_nam_files[0].filename), '') for f in myzipfile.infolist(): if f.is_dir(): # folder continue elif f.filename[0] == '_': # private continue elif os.path.basename(f.filename)[0] == '.': # private continue elif f.filename[:len(nam_dir)] == nam_dir: old_name = f.filename f.filename = f.filename[len(nam_dir):] new_name = f.filename s = 'Copying {0:s} to {1:s}'.format(f.filename, temp_dir) print(s) myzipfile.extract(f, temp_dir) # location to nam file. Always on top of directory nam_fp = os.path.join(temp_dir, os.path.basename(all_nam_files[0].filename)) assert os.path.isfile(nam_fp), f'{nam_fp} does not exist' mp = Model(nam_fp) nb = mp.script_model2nb(use_yapf=False) buff = io.BytesIO() zip_out = zipfile.ZipFile(buff, 'w') ipynb_buff = io.StringIO(nbformat.writes(nb)) zip_out.writestr('test.ipynb', ipynb_buff.getvalue()) for out_formats_item, out_formats_ext_item in zip( out_formats, out_formats_ext): ipynb_buff.seek(0) zip_out.writestr( 'test.' + out_formats_ext_item, nbconvert.export(nbconvert.get_exporter(out_formats_item), ipynb_buff)[0]) ipynb_buff.close() zip_out.close() print('\nContent of the zipfile:') zip_out.printdir() return buff
try: from flopymetascript.model import Model import io import nbformat import zipfile import nbconvert import flopy p_list = list(flopy.seawat.Seawat().mfnam_packages.keys()) mp = Model(add_pack=p_list) # with description nb = mp.script_model2nb(use_yapf=False) ipynb_buff = io.StringIO(nbformat.writes(nb)) s = nbconvert.export(nbconvert.get_exporter('markdown'), ipynb_buff)[0] s = s.replace('\n\n```', '\n\n```python') p_list_order = list(mp.packages.keys()) toc = [ '* [' + i + '](#' + i.replace('.', '') + ')\n' for i in p_list_order ] toc = ''.join(toc) with open('wiki_default_parameters.md', 'w') as f: f.write(toc) f.write(s) ipynb_buff.close()
notebook, pdf, python, rst, script, slides, webpdf. (default: notebook) - exporter_kwargs (dict, optional): The arguments used for initializing the exporter. ======= - output_format (str, optional): Notebook output format. Currently supported: json, html (default: json) >>>>>>> prefect clone """ nb: nbformat.NotebookNode = pm.execute_notebook( path, "-", parameters=parameters, kernel_name=self.kernel_name ) if output_format == "json": <<<<<<< HEAD output_format = "notebook" if exporter_kwargs is None: exporter_kwargs = {} exporter = nbconvert.get_exporter(output_format) body, resources = nbconvert.export(exporter, nb, **exporter_kwargs) return body ======= return nbformat.writes(nb) if output_format == "html": html_exporter = nbconvert.HTMLExporter() (body, resources) = html_exporter.from_notebook_node(nb) return body raise NotImplementedError("Notebook output %s not supported", output_format) >>>>>>> prefect clone
def check_files(ipynbfile, pyfile, verbose=True, prefix="FAIL:"): """ This method checks a notebook if there are any general errors in it. After doing this it will take a python file that contains assert statements and will throw a status code of 2 if there are failures. Will also print the prefixed error message. :param ipynbfile: the notebook file :param pyfile: the python file :param verbose: turn on verbose mode, will print the code in both files :param prefix: optional prefix to add to the error message """ if os.path.splitext(ipynbfile)[1] != '.ipynb': raise ValueError(".ipynb file needs to have .ipynb extension") if os.path.splitext(pyfile)[1] != '.py': raise ValueError(".py file needs to have .py extension") with open(ipynbfile, 'r') as readfile: notebook = nbformat.read(readfile, as_version=nbformat.NO_CONVERT) output, metadata = nbconvert.export(nbconvert.PythonExporter, notebook) if verbose: for i, cell in enumerate(output.split("#")): code = "#" + cell click.echo(click.style(f'showing block {i}', fg='blue')) for line in code.split("\n"): if line != "\n": print(" " + line) try: exec(code) except: click.echo(click.style(f"Error in codeblock!", fg='red')) traceback.print_exc() click.echo( click.style( f"{prefix}notebook contains error, could not even pass it to py-file.", fg='red')) sys.exit(2) click.echo(click.style(f"Notebook {ipynbfile} has been parsed.", fg='blue')) try: exec(output) click.echo( click.style(f"Notebook {ipynbfile} has been executed.", fg='blue')) except: click.echo(click.style("ERROR!", fg="red")) click.echo( f"{prefix} Jupyter notebook gave an error! Make sure it runs without error." ) sys.exit(2) if verbose: click.echo(click.style(f"Showing contents of {pyfile}.", fg='blue')) with open(pyfile, "r") as f: for l in f.readlines(): print(l.replace("\n", "")) tmpfile = "tmpfile.py" with open(tmpfile, "a") as f: f.write(output) with open(pyfile, "r") as py: f.write(py.read()) click.echo( click.style("Temporary file made. About to make test.", fg='blue')) try: exec(open(tmpfile).read()) os.remove(tmpfile) except AssertionError as e: click.echo(click.style("ASSERTION ERROR DETECTED:", fg='red')) print(f"{prefix} {e}") os.remove(tmpfile) sys.exit(2)
""" This script is called by sphinxdoc to convert all Jupyter notebooks in the Examples folder to ReStructured Text files and stores them in the documentation folder. Author: R.A. Collenteur 2016 """ import nbconvert from nbconvert import RSTExporter import glob import io import os notebooks = glob.iglob('../examples/notebooks/*.ipynb') for nb in notebooks: (x,resources) = nbconvert.export(RSTExporter, nb) with io.open('../doc/examples/%s.rst' % os.path.basename(nb)[:-6], 'w', encoding='utf-8') as f: f.write(x[:]) for fname in resources['outputs'].keys(): with io.open('../doc/examples/%s' % fname, 'wb') as f: f.write(resources['outputs'][fname]) print("example notebooks successfully converted to .rst files")