def export_notebook(nb, nb_path, output_dir, SCOPETYPE=None, PLATFORM=None): """Takes a notebook node and exports it to ReST and HTML Args: nb (notebook): The notebook returned by execute_notebook. nb_path (str): Path to intput notebook file. Used to generate the name of the output file. output_dir (str): The output directory for the ReST and HTML file. SCOPETYPE (str): Used to generate the output file name. PLATFORM (str): Used to generate the output file name. """ notebook_dir, file_name = os.path.split(nb_path) file_name_root, _ = os.path.splitext(file_name) base_path = os.path.join( output_dir, file_name_root + "-{}-{}".format(SCOPETYPE, PLATFORM)) rst_path = os.path.abspath(base_path + ".rst") html_path = os.path.abspath(base_path + ".html") with open(rst_path, "w", encoding='utf-8') as rst_file: rst_exporter = RSTExporter() body, res = rst_exporter.from_notebook_node(nb) rst_file.write(body) print('Wrote to: ', rst_path) with open(html_path, 'w', encoding='utf-8') as html_file: html_exporter = HTMLExporter() body, res = html_exporter.from_notebook_node(nb) html_file.write(body) print('Wrote to: ', html_path)
def notebook_object_to_rst(ntbk, rpth, cr=None): """ Convert notebook object `ntbk` to rst document at `rpth`, in directory `rdir`. Parameter `cr` is a CrossReferenceLookup object. """ # Parent directory of file rpth rdir = os.path.dirname(rpth) # File basename rb = os.path.basename(os.path.splitext(rpth)[0]) # Pre-process notebook prior to conversion to rst if cr is not None: preprocess_notebook(ntbk, cr) # Convert notebook to rst rex = RSTExporter() rsttxt, rstres = rex.from_notebook_node(ntbk) # Replace `` with ` in sphinx cross-references rsttxt = re.sub(r':([^:]+):``(.*?)``', r':\1:`\2`', rsttxt) # Insert a cross-reference target at top of file reflbl = '.. _examples_' + os.path.basename(rdir) + '_' + \ rb.replace('-', '_') + ':\n' rsttxt = reflbl + rsttxt # Write the converted rst to disk write_notebook_rst(rsttxt, rstres, rb, rdir)
def notebook_object_to_rst(ntbk, rpth, cr=None): """ Convert notebook object `ntbk` to rst document at `rpth`, in directory `rdir`. Parameter `cr` is a CrossReferenceLookup object. """ # Parent directory of file rpth rdir = os.path.dirname(rpth) # File basename rb = os.path.basename(os.path.splitext(rpth)[0]) # Pre-process notebook prior to conversion to rst if cr is not None: preprocess_notebook(ntbk, cr) # Convert notebook to rst rex = RSTExporter() rsttxt, rstres = rex.from_notebook_node(ntbk) # Replace `` with ` in sphinx cross-references rsttxt = re.sub(r':([^:]+):``(.*?)``', r':\1:`\2`', rsttxt) # Insert a cross-reference target at top of file reflbl = '.. _examples_' + os.path.basename(rdir) + '_' + \ rb.replace('-', '_') + ':\n\n' rsttxt = reflbl + rsttxt # Write the converted rst to disk write_notebook_rst(rsttxt, rstres, rb, rdir)
def nb2rst(filepath): with open(filepath) as fh: nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT) exporter = RSTExporter() # source is a tuple of python source code # meta contains metadata source, meta = exporter.from_notebook_node(nb) return [line + '\n' for line in source.split('\n')]
def __init__(self,path,name): fname = os.path.join(path, name) with open(fname, "r") as f: self.json = f.read() self.nb = nbformat.reads(self.json, as_version=4) basename = os.path.splitext(name)[0] self.outfname = os.path.join(path,'ref',basename + ".new") self.refname = os.path.join(path,'ref',basename + ".md") self.exp = RSTExporter()
def export_notebook(nb, nb_path, output_dir, SCOPETYPE=None, PLATFORM=None): """Takes a notebook node and exports it to ReST and HTML Args: nb (notebook): The notebook returned by execute_notebook. nb_path (str): Path to intput notebook file. Used to generate the name of the output file. output_dir (str): The output directory for the ReST and HTML file. SCOPETYPE (str): Used to generate the output file name. PLATFORM (str): Used to generate the output file name. """ notebook_dir, file_name = os.path.split(nb_path) #need to make sure course is in rst file name notebook_dir = notebook_dir.replace(r'\\', '_').replace('/', '_') file_name_root, _ = os.path.splitext(notebook_dir + '_' + file_name) base_path = os.path.join( output_dir, file_name_root + '-{}-{}'.format(SCOPETYPE, PLATFORM)) rst_path = os.path.abspath(base_path + '.rst') html_path = os.path.abspath(base_path + '.html') # class EscapeBacktickPreprocessor(nbconvert.preprocessors.Preprocessor): ebp = EscapeBacktickPreprocessor() rst_ready_nb, _ = ebp.preprocess(nb, {}) with open(rst_path, 'w', encoding='utf-8') as rst_file: rst_exporter = RSTExporter() body, res = rst_exporter.from_notebook_node( rst_ready_nb, resources={ 'unique_key': 'img/{}-{}-{}'.format(SCOPETYPE, PLATFORM, file_name_root).replace(' ', '') }) file_names = res['outputs'].keys() for name in file_names: with open(os.path.join(output_dir, name), 'wb') as f: f.write(res['outputs'][name]) print('writing to ', name) #print(res['outputs'][name]) rst_file.write(body) print('Wrote to: ', rst_path) ## need resources with open(html_path, 'w', encoding='utf-8') as html_file: html_exporter = HTMLExporter() body, res = html_exporter.from_notebook_node(nb) html_file.write(body) print('Wrote to: ', html_path)
def test_codefolding(): """ Test codefolding preprocessor """ nb_name='tests/data/codefolding.ipynb' with open(nb_name, 'r') as f: notebook_json = f.read() notebook = nbformat.reads(notebook_json, as_version=4) c.RSTExporter.preprocessors = ["pre_codefolding.CodeFoldingPreprocessor"] c.NbConvertApp.export_format = 'rst' rst_exporter = RSTExporter(config=c) body = rst_exporter.from_notebook_node(notebook) assert 'AXYZ12AXY' not in body[0] # firstline fold assert 'GR4CX32ZT' not in body[0] # function fold
def test_codefolding(): """ Test codefolding preprocessor """ nb_name = 'tests/data/codefolding.ipynb' with open(nb_name, 'r') as f: notebook_json = f.read() notebook = nbformat.reads(notebook_json, as_version=4) c.RSTExporter.preprocessors = ["pre_codefolding.CodeFoldingPreprocessor"] c.NbConvertApp.export_format = 'rst' rst_exporter = RSTExporter(config=c) body = rst_exporter.from_notebook_node(notebook) assert 'AXYZ12AXY' not in body[0] # firstline fold assert 'GR4CX32ZT' not in body[0] # function fold
def test_pymarkdown_preprocessor(): """ Test python markdown preprocessor """ nb_name = 'tests/data/pymarkdown.ipynb' with open(nb_name, 'r') as f: notebook_json = f.read() notebook = nbformat.reads(notebook_json, as_version=4) c.RSTExporter.preprocessors = ["pre_pymarkdown.PyMarkdownPreprocessor"] c.NbConvertApp.export_format = 'rst' rst_exporter = RSTExporter(config=c) body = rst_exporter.from_notebook_node(notebook) with open('test.txt', 'wb') as f: f.write(body[0].encode('utf8')) assert 'Hello world' in body[0] pass
def test_pymarkdown_preprocessor(): """ Test python markdown preprocessor """ nb_name='tests/data/pymarkdown.ipynb' with open(nb_name, 'r') as f: notebook_json = f.read() notebook = nbformat.reads(notebook_json, as_version=4) c.RSTExporter.preprocessors = ["pre_pymarkdown.PyMarkdownPreprocessor"] c.NbConvertApp.export_format = 'rst' rst_exporter = RSTExporter(config=c) body = rst_exporter.from_notebook_node(notebook) with open('test.txt', 'wb') as f: f.write(body[0].encode('utf8')) assert 'Hello world' in body[0] pass
def create_notebooks(notebooks): from nbconvert import RSTExporter import io import nbformat rst_exporter = RSTExporter() for filename, name in notebooks: head, tail = os.path.split(filename) print("\tgenerate {0}.rst".format(name)) with io.open(filename, encoding='utf-8') as source: notebook = nbformat.reads(source.read(), as_version=4) (body, resources) = rst_exporter.from_notebook_node(notebook) body = body.replace(".. code:: python", ".. code-block:: python") with io.open(filename.replace('.ipynb', '.rst'), "w", encoding='utf-8') as target: target.write(body) path = os.path.split(filename)[0] print(path) for k, v in resources['outputs'].items(): with io.open(path + '/' + k, "wb") as target: target.write(v)
def convert(nbfile): basename, _ = os.path.splitext(nbfile) meta = {"metadata": {"path": "."}} with open(nbfile, "r", encoding="utf-8") as nbf: nbdata = nbformat.read(nbf, as_version=4, encoding="utf-8") runner = ExecutePreprocessor(timeout=600, kernel_name="probscale") runner.preprocess(nbdata, meta) img_folder = basename + "_files" body_raw, images = RSTExporter().from_notebook_node(nbdata) body_final = body_raw.replace(".. image:: ", ".. image:: {}/".format(img_folder)) with open(basename + ".rst", "w", encoding="utf-8") as rst_out: rst_out.write(body_final) for img_name, img_data in images["outputs"].items(): img_path = os.path.join(img_folder, img_name) with open(img_path, "wb") as img: img.write(img_data)
def convert(nbfile): basename, _ = os.path.splitext(nbfile) meta = {'metadata': {'path': '.'}} with open(nbfile, 'r', encoding='utf-8') as nbf: nbdata = nbformat.read(nbf, as_version=4, encoding='utf-8') runner = ExecutePreprocessor(timeout=600, kernel_name='probscale') runner.preprocess(nbdata, meta) img_folder = basename + '_files' body_raw, images = RSTExporter().from_notebook_node(nbdata) body_final = body_raw.replace('.. image:: ', '.. image:: {}/'.format(img_folder)) with open(basename + '.rst', 'w', encoding='utf-8') as rst_out: rst_out.write(body_final) for img_name, img_data in images['outputs'].items(): img_path = os.path.join(img_folder, img_name) with open(img_path, 'wb') as img: img.write(img_data)
def parse_notebook_index(ntbkpth): """ Parse the top-level notebook index file at `ntbkpth`. Returns a list of subdirectories in order of appearance in the index file, and a dict mapping subdirectory name to a description. """ # Convert notebook to RST text in string rex = RSTExporter() rsttxt = rex.from_filename(ntbkpth)[0] # Clean up trailing whitespace rsttxt = re.sub(r'\n ', r'', rsttxt, re.M | re.S) pthidx = {} pthlst = [] lines = rsttxt.split('\n') for l in lines: m = re.match(r'^-\s+`([^<]+)\s+<([^>]+).ipynb>`__', l) if m: # List of subdirectories in order of appearance in index.rst pthlst.append(m.group(2)) # Dict mapping subdirectory name to description pthidx[m.group(2)] = m.group(1) return pthlst, pthidx
def test_nbconvert_demo(self): self.maxDiff = None with open(TESTS_NB_FILE) as f: notebook = nbformat.read(f, as_version=4) c = Config() c.MemestraDeprecationChecker.decorator = ('decoratortest', 'deprecated') c.RSTExporter.preprocessors = [preprocessor.MemestraDeprecationChecker] deprecation_checker = RSTExporter(config=c) # the RST exporter behaves differently on windows and on linux # there can be some lines with only whitespaces # so we ignore differences that only consist of empty lines rst = deprecation_checker.from_notebook_node(notebook)[0] lines = rst.split('\n') lines = [l.rstrip() for l in lines] rst = '\n'.join(lines) with open(os.path.join(this_dir, 'misc', 'memestra_nb_demo.rst')) as f: rst_true = f.read() self.assertEqual(rst, rst_true)
def convert_to_rst(notebook, cache=True): notebook_mtime = path.getmtime(notebook) path_dir = path.dirname(notebook) cache_file = path.join(path_dir, '.{}_{}'.format( notebook_mtime, path.basename(notebook))) if path.exists(cache_file): print('Reading from cache: ', cache_file) with open(cache_file) as f: body = f.read() if body == '': remove(cache_file) return convert_to_rst(notebook, cache) else: # Instantiate it rst_exporter = RSTExporter() # Convert the notebook to RST format (body, resources) = rst_exporter.from_filename(notebook) print('Writing to cache: ', cache_file) with open(cache_file, 'w') as f: f.write(body) return body
def __init__(self, output): assert output in SUPPORTED_FORMATS, f"supported formats are {SUPPORTED_FORMATS}" self.read_encoding = "utf-8" self.write_encoding = "utf-8" self.format = output if self.format == "pdf": pdf = PDFExporter() pdf.exclude_output_prompt = True pdf.exclude_input = True self.exporter = pdf elif self.format == "rst": self.exporter = RSTExporter() else: self.exporter = MarkdownExporter()
def __init__(self, output_format, destination_mode): assert output_format in SUPPORTED_FORMATS, f"supported formats are {SUPPORTED_FORMATS}" assert ( destination_mode in SUPPORTED_DESTINATIONS_MODES ), f"supported destination modes are {SUPPORTED_DESTINATIONS_MODES}" self.read_encoding = "utf-8" self.write_encoding = "utf-8" self.format = output_format self.destination_mode = destination_mode if self.format == "pdf": pdf = PDFExporter() pdf.exclude_output_prompt = True pdf.exclude_input = True self.exporter = pdf elif self.format == "rst": self.exporter = RSTExporter() else: self.exporter = MarkdownExporter()
def export_rst(jupyter_output, output_notebook, add_nunit_attachment, file_ext='.rst', root="."): """ Export Jupyter Output to RST File :param jupyter_output: :param output_notebook: :param add_nunit_attachment: :param file_ext: :param root: """ rst_exporter = RSTExporter() export_notebook(rst_exporter, jupyter_output, output_notebook, add_nunit_attachment, file_ext, root=root)
class Tester(object): def __init__(self,path,name): fname = os.path.join(path, name) with open(fname, "r") as f: self.json = f.read() self.nb = nbformat.reads(self.json, as_version=4) basename = os.path.splitext(name)[0] self.outfname = os.path.join(path,'ref',basename + ".new") self.refname = os.path.join(path,'ref',basename + ".md") self.exp = RSTExporter() #self.exp.template_file = 'basic' def runtest(self): warnings.filterwarnings("ignore",category=DeprecationWarning) ep = ExecutePreprocessor(timeout=600, kernel_name='sagemath',allow_errors=True) ep.preprocess(self.nb, {}) (body, resources) = self.exp.from_notebook_node(self.nb) with open(self.outfname,'wt') as f: f.write(body) if not filecmp.cmp(self.outfname,self.refname): raise ValueError, "files %s and %s differ" % (self.outfname,self.refname)
def convert_to_rst(nb, basedir, fpath, fstem): # Convert to .rst formats exp = RSTExporter() c = Config() c.TagRemovePreprocessor.remove_cell_tags = {"hide"} c.TagRemovePreprocessor.remove_input_tags = {"hide-input"} c.TagRemovePreprocessor.remove_all_outputs_tags = {"hide-output"} c.ExtractOutputPreprocessor.output_filename_template = ( f"{fstem}_files/{fstem}_" + "{cell_index}_{index}{extension}" ) exp.register_preprocessor(TagRemovePreprocessor(config=c), True) exp.register_preprocessor(ExtractOutputPreprocessor(config=c), True) body, resources = exp.from_notebook_node(nb) # Clean the output on the notebook and save a .ipynb back to disk print(f"Writing clean {fpath} ... ", end=" ", flush=True) nb = strip_output(nb) with open(fpath, "wt") as f: nbformat.write(nb, f) # Write the .rst file rst_path = os.path.join(basedir, f"{fstem}.rst") print(f"Writing {rst_path}") with open(rst_path, "w") as f: f.write(body) print(resources["outputs"]) # Write the individual image outputs imdir = os.path.join(basedir, f"{fstem}_files") if not os.path.exists(imdir): os.mkdir(imdir) for imname, imdata in resources["outputs"].items(): if imname.startswith(fstem): impath = os.path.join(basedir, f"{imname}") with open(impath, "wb") as f: f.write(imdata) f.write(imdata)
dst = targetfolder / "copyright.pandoc" print(f"* Copying copyright to {str(targetfolder)!r} ...") shutil.copyfile(src, dst) finally: os.chdir(cur_wd) shutil.rmtree(tempfolder) download_pandoc() # Import the RST exporter and instantiate it # 3rd party from nbconvert import RSTExporter rst_exporter = RSTExporter() replacements = { "pyms.GCMS.IO.JCAMP": ":mod:`pyms.GCMS.IO.JCAMP`", "pyms.GCMS.IO.ANDI": ":mod:`pyms.GCMS.IO.ANDI`", "pyms.GCMS.Class.GCMS_data": ":class:`pyms.GCMS.Class.GCMS_data`", "GCMS_data": ":class:`~pyms.GCMS.Class.GCMS_data`", "pyms.Spectrum.Scan": ":class:`pyms.Spectrum.Scan`", "Scan": ":class:`~pyms.Spectrum.Scan`", "GCMS_data.tic":
# stdlib import re # 3rd party import nbformat from domdf_python_tools.paths import PathPlus from nbconvert import RSTExporter rst_exporter = RSTExporter() replacements = { # "pyms.GCMS.IO.JCAMP": # ":mod:`pyms.GCMS.IO.JCAMP`", # "pyms.GCMS.IO.ANDI": # ":mod:`pyms.GCMS.IO.ANDI`", # "pyms.GCMS.Class.GCMS_data": # ":class:`pyms.GCMS.Class.GCMS_data`", # "GCMS_data": # ":class:`~pyms.GCMS.Class.GCMS_data`", # "pyms.Spectrum.Scan": # ":class:`pyms.Spectrum.Scan`", # "Scan": # ":class:`~pyms.Spectrum.Scan`", # "GCMS_data.tic": # ":attr:`GCMS_data.tic <pyms.GCMS.Class.GCMS_data.tic>`", # "IonChromatogram": # ":class:`~pyms.IonChromatogram.IonChromatogram`", # "info()": # ":py:meth:`info() <pyms.GCMS.Class.GCMS_data.info()>`", # "write()": # ":py:meth:`write() <pyms.GCMS.Class.GCMS_data.write()>`",
ep.preprocess(nb, {"metadata": {"path": basedir}}) # Remove plain text execution result outputs for cell in nb.get("cells", {}): fields = cell.get("outputs", []) for field in fields: if field["output_type"] == "execute_result": data_keys = field["data"].keys() for key in list(data_keys): if key == "text/plain": field["data"].pop(key) if not field["data"]: fields.remove(field) # Convert to .rst formats exp = RSTExporter() c = Config() c.TagRemovePreprocessor.remove_cell_tags = {"hide"} c.TagRemovePreprocessor.remove_input_tags = {"hide-input"} c.TagRemovePreprocessor.remove_all_outputs_tags = {"hide-output"} c.ExtractOutputPreprocessor.output_filename_template = \ f"{fstem}_files/{fstem}_" + "{cell_index}_{index}{extension}" exp.register_preprocessor(TagRemovePreprocessor(config=c), True) exp.register_preprocessor(ExtractOutputPreprocessor(config=c), True) body, resources = exp.from_notebook_node(nb) # Clean the output on the notebook and save a .ipynb back to disk print(f"Writing clean {fpath} ... ", end=" ", flush=True)
Convert ipynb notebook to rst """ # stdlib import io import pathlib # Install pandoc from pandoc_downloader import download_pandoc download_pandoc() # Import the RST exproter from nbconvert import RSTExporter # Instantiate it rst_exporter = RSTExporter() # from nbconvert.preprocessors import ExecutePreprocessor # ep = ExecutePreprocessor(timeout=600, kernel_name='python3') # rst_exporter.register_preprocessor(ep, True) replacements = { "pyms.GCMS.IO.JCAMP": ":mod:`pyms.GCMS.IO.JCAMP`", "pyms.GCMS.IO.ANDI": ":mod:`pyms.GCMS.IO.ANDI`", "pyms.GCMS.Class.GCMS_data": ":class:`pyms.GCMS.Class.GCMS_data`", "GCMS_data": ":class:`~pyms.GCMS.Class.GCMS_data`", "pyms.Spectrum.Scan": ":class:`pyms.Spectrum.Scan`",
for notebook in notebooks: try: f = open(notebook, 'rt') example_nb = f.read() f.close() rst_path = os.path.join(cwd, 'doc', 'source') path_parts = os.path.split(notebook) nb_filename = path_parts[-1] nb_filename = nb_filename.split('.')[0] source_dir = nb_filename.split('_')[0] rst_filename = os.path.join(cwd, 'doc', 'source', source_dir, nb_filename + '.rst') example_nb = nbformat.reader.reads(example_nb) rst_export = RSTExporter() (body, resources) = rst_export.from_notebook_node(example_nb) with open(rst_filename, 'wt') as rst: rst.write(body) for key in resources['outputs'].keys(): if key.endswith('.png'): resource_filename = os.path.join(cwd, 'doc', 'source', source_dir, key) with open(resource_filename, 'wb') as resource: resource.write(resources['outputs'][key]) except: import warnings warnings.warn('Unable to convert {original} to {target}. This '
notebooks = glob.glob(os.path.join(cwd, 'examples', '*.ipynb')) for notebook in notebooks: try: with open(notebook, 'rt') as f: example_nb = f.read() rst_path = os.path.join(cwd, 'doc', 'source') path_parts = os.path.split(notebook) nb_filename = path_parts[-1] nb_filename = nb_filename.split('.')[0] source_dir = nb_filename.split('_')[0] rst_filename = os.path.join(cwd, 'doc', 'source', source_dir, nb_filename + '.rst') example_nb = nbformat.reader.reads(example_nb) rst_export = RSTExporter() (body, resources) = rst_export.from_notebook_node(example_nb) with open(rst_filename, 'wt') as rst: rst.write(body) for key in resources['outputs'].keys(): if key.endswith('.png'): resource_filename = os.path.join(cwd, 'doc', 'source', source_dir, key) with open(resource_filename, 'wb') as resource: resource.write(resources['outputs'][key]) except: import warnings warnings.warn('Unable to convert {original} to {target}. This '
def __init__(self): self.exporter = RSTExporter()