def convert_notebook(nb_path): prt('Convert notebook to Metatab source package') if not exists(nb_path): err("Notebook path does not exist: '{}' ".format(nb_path)) c = Config() pe = NotebookExecutor(config=c, log=logger) prt('Running the notebook') output, resources = pe.from_filename(nb_path) fw = FilesWriter() fw.build_directory = pe.output_dir fw.write(output, resources, notebook_name=DEFAULT_METATAB_FILE) de = DocumentationExporter(config=c, log=logger, metadata=doc_metadata(pe.doc)) prt('Exporting documentation') output, resources = de.from_filename(nb_path) fw.build_directory = join(pe.output_dir, 'docs') fw.write(output, resources, notebook_name='notebook') new_mt_file = join(pe.output_dir, DEFAULT_METATAB_FILE) doc = MetapackDoc(new_mt_file) de.update_metatab(doc, resources) for lib_dir in pe.lib_dirs: lib_dir = normpath(lib_dir).lstrip('./') doc['Resources'].new_term("Root.PythonLib", lib_dir) path = abspath(lib_dir) dest = join(pe.output_dir, lib_dir) ensure_dir(dest) copytree(path, join(pe.output_dir, lib_dir)) doc.write_csv() # Reset the input to use the new data prt('Running with new package file: {}'.format(new_mt_file))
def notebook_to_rst(nbfilename): nbfilepath = os.path.join(EXPATH, nbfilename) rstfilename = get_rstfilename(nbfilename) output_files_dir = only_filename_no_ext(rstfilename) metadata_path = os.path.dirname(rstfilename) unique_key = nbfilename.rstrip('.ipynb') resources = { 'metadata': {'path': metadata_path}, 'output_files_dir': output_files_dir, # Prefix for the output image filenames 'unique_key': unique_key } # Read notebook with open(nbfilepath, 'r') as f: nb = nbformat.read(f, as_version=4) # Export exporter = nbsphinx.Exporter(execute='never', allow_errors=True) (body, resources) = exporter.from_notebook_node(nb, resources) # Correct path for the resources for filename in list(resources['outputs'].keys()): tmp = os.path.join(RST_PATH, filename) resources['outputs'][tmp] = resources['outputs'].pop(filename) fw = FilesWriter() fw.build_directory = RST_PATH # Prevent "not in doctree" complains resources['output_extension'] = '' body = 'Examples\n--------\n' + body fw.write(body, resources, notebook_name=rstfilename)
def notebook_to_rst(nbfilename): nbfilepath = os.path.join(EXPATH, nbfilename) rstfilename = get_rstfilename(nbfilename) output_files_dir = only_filename_no_ext(rstfilename) metadata_path = os.path.dirname(rstfilename) unique_key = nbfilename.rstrip('.ipynb') resources = { 'metadata': {'path': metadata_path}, 'output_files_dir': output_files_dir, # Prefix for the output image filenames 'unique_key': unique_key } # Read notebook with open(nbfilepath, 'r') as f: nb = nbformat.read(f, as_version=4) # Export exporter = nbsphinx.Exporter(execute='never', allow_errors=True) (body, resources) = exporter.from_notebook_node(nb, resources) # Correct path for the resources for filename in list(resources['outputs'].keys()): tmp = os.path.join(RST_PATH, filename) resources['outputs'][tmp] = resources['outputs'].pop(filename) fw = FilesWriter() fw.build_directory = RST_PATH # Prevent "not in doctree" complains resources['output_extension'] = '' body = 'Examples\n--------\n' + body fw.write(body, resources, notebook_name=rstfilename)
def convert_documentation(nb_path): """Run only the document conversion portion of the notebook conversion The final document will not be completel """ with open(nb_path) as f: nb = nbformat.reads(f.read(), as_version=4) doc = ExtractInlineMetatabDoc(package_url="metapack+file:" + dirname(nb_path)).run(nb) package_name = doc.as_version(None) output_dir = join(getcwd(), package_name) de = DocumentationExporter(config=Config(), log=logger, metadata=doc_metadata(doc)) prt('Converting documentation') output, resources = de.from_filename(nb_path) fw = FilesWriter() fw.build_directory = join(output_dir, 'docs') fw.write(output, resources, notebook_name='notebook') prt("Wrote documentation to {}".format(fw.build_directory))
def _convert(self, tmpdir: Path, entry: Path, outdir: Path, depth: int): """Convert a notebook. Args: tmpdir: Temporary working directory entry: notebook to convert outdir: output directory for .html and .rst files depth: depth below root, for fixing image paths """ test_mode = self.s.get("test_mode") # strip special cells. if self._has_tagged_cells(entry, set(self._cell_tags.values())): _log.debug(f"notebook '{entry.name}' has test cell(s)") orig_entry, entry = entry, self._strip_tagged_cells( tmpdir, entry, ("remove", "exercise"), "testing") notify(f"Stripped tags from: {orig_entry.name}", 3) else: # copy to temporary directory just to protect from output cruft tmp_entry = tmpdir / entry.name shutil.copy(entry, tmp_entry) orig_entry, entry = entry, tmp_entry # convert all tag-stripped versions of the notebook. # before running, check if converted result is newer than source file if self._already_converted(orig_entry, entry, outdir): notify( f"Skip notebook conversion, output is newer, for: {entry.name}", 3) self._results.cached.append(entry) return notify(f"Running notebook: {entry.name}", 3) nb = self._parse_and_execute(entry) if test_mode: # don't do conversion in test mode return notify(f"Exporting notebook '{entry.name}' to directory {outdir}", 3) wrt = FilesWriter() # export each notebook into multiple target formats created_wrapper = False for (exp, postprocess_func, pp_args) in ( (RSTExporter(), self._postprocess_rst, ()), (HTMLExporter(), self._postprocess_html, (depth, )), ): _log.debug( f"export '{orig_entry}' with {exp} to notebook '{entry}'") (body, resources) = exp.from_notebook_node(nb) body = postprocess_func(body, *pp_args) wrt.build_directory = str(outdir) wrt.write(body, resources, notebook_name=entry.stem) # create a 'wrapper' page if not created_wrapper: _log.debug( f"create wrapper page for '{entry.name}' in '{outdir}'") self._create_notebook_wrapper_page(entry.stem, outdir) created_wrapper = True # move notebooks into docs directory _log.debug(f"move notebook '{entry} to output directory: {outdir}") shutil.copy(entry, outdir / entry.name)
def convert(self, file): assert os.path.exists( file), f"this should not happen, path {file} must exist" body, resources = self.export(file) fw = FilesWriter() fw.build_directory = os.path.dirname(file) f_name = os.path.basename(file).replace(".ipynb", "") fw.write(body, resources, notebook_name=f_name)
def convert(self, file): assert os.path.exists( file), f"this should not happen, path {file} must exist" body, resources = self.export(file) fw = FilesWriter() fw._makedir(self.dst_folder(file)) fw.build_directory = self.dst_folder(file) fw.write(body, resources, notebook_name=self.dest_file(file, withFormat=False))
def export(filename): c = Config() # Configure our tag removal c.TagRemovePreprocessor.remove_cell_tags = ("hide_cell", ) c.TagRemovePreprocessor.remove_all_outputs_tags = ('hide_output', ) c.TagRemovePreprocessor.remove_input_tags = ('hide_input', ) # Configure and run out exporter c.LatexExporter.preprocessors = [ 'nbconvert.preprocessors.TagRemovePreprocessor' ] targetDirectoryFull_loc = targetDirectoryFull + "\\" + filename.replace( ' ', '_')[:-6] targetFolder = filename.replace(' ', '_')[:-6] writer = FilesWriter() writer.build_directory = targetDirectoryFull_loc test = writer.write(*CustomLatexExporter(config=c).from_filename(filename), notebook_name=filename[:-6]) createdFile = targetDirectoryFull_loc + '\\' + filename[:-6] + '.tex' toBeCreatedFile = targetDirectoryCore + '\\' + filename[:-6] + '.tex' coreFileName = targetDirectoryCore + '\\' + filename[:-6] + '_Core.tex' tag = 'maketitle' lines_to_write = [] tag_found = False with open(createdFile, encoding="utf8") as in_file: for line in in_file: if tag in line: tag_found = True elif tag_found: line = re.sub(r"{In}{incolor}{\d*}", r"{In}{incolor}{In}", line) line = re.sub(r"{Out}{outcolor}{\d*}", r"{Out}{outcolor}{Out}", line) string = r'JupyterNotebooksFull/' + targetFolder + '/' line = re.sub(r"(output.*jpg})", string + r"\g<1>", line) if r'\prompt{Out}{outcolor}{Out}{}' in line: lines_to_write.append( r' \begin{tcolorbox}[breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0]' + '\n') line = r'\prompt{Out}{outcolor}{Out}{\boxspacing}' lines_to_write.append(line) if r'{ \hspace*{\fill} \\}' in line: lines_to_write.append(r'\end{tcolorbox}' + '\n') lines_to_write = lines_to_write[1:-5] with open(toBeCreatedFile, 'w+', encoding="utf8") as out_file: out_file.writelines(lines_to_write)
def _load_documentation_files(self): from metapack_jupyter.exporters import DocumentationExporter notebook_docs = [] # First find and remove notebooks from the docs. These wil get processed to create # normal documents. try: for term in list( self.doc['Documentation'].find('Root.Documentation')): u = parse_app_url(term.value) if u is not None and u.target_format == 'ipynb' and u.proto == 'file': notebook_docs.append(term) self.doc.remove_term(term) except KeyError: self.warn("No documentation defined in metadata") # Process all of the normal files super()._load_documentation_files() fw = FilesWriter() fw.build_directory = join(self.package_path.path, 'docs') # Now, generate the notebook documents directly into the filesystem package for term in notebook_docs: de = DocumentationExporter( base_name=term.name or slugify(term.title)) u = parse_app_url(term.value) nb_path = join(self.source_dir, u.path) # Only works if the path is relative. try: output, resources = de.from_filename(nb_path) fw.write(output, resources, notebook_name=de.base_name + '_full') # Write notebook html with inputs de.update_metatab(self.doc, resources) except Exception as e: from metapack.cli.core import warn warn("Failed to convert document for {}: {}".format( term.name, e))
def convert_to_html(result_notebook: Path) -> Path: """ :param result_notebook: The path to the result notebook :return: Path with output extension """ print(f"Running conversion to HTML for {result_notebook}") with result_notebook.open() as f: notebook = nbformat.read(f, as_version=4) html_exporter = HTMLExporter() html_exporter.exclude_input = True (body, resources) = html_exporter.from_notebook_node(notebook) write_file = FilesWriter() write_file.build_directory = str(result_notebook.parent) write_file.write(output=body, resources=resources, notebook_name=str(result_notebook.stem)) return result_notebook.with_suffix(resources['output_extension'])
def generate_notebook(template_notebook: Path, notebook_params: Dict, result_notebook: Path) -> Path: print(f"Writing report to {result_notebook}") papermill.execute_notebook(input_path=str(template_notebook), output_path=str(result_notebook), parameters=notebook_params, progress_bar=False) print(f"Running conversion to HTML for {result_notebook}") with result_notebook.open() as f: notebook = nbformat.read(f, as_version=4) html_exporter = HTMLExporter() html_exporter.exclude_input = True (body, resources) = html_exporter.from_notebook_node(notebook) write_file = FilesWriter() write_file.build_directory = str(result_notebook.parent) write_file.write(output=body, resources=resources, notebook_name=str(result_notebook.stem)) return result_notebook.with_suffix(resources['output_extension'])
def strip_tags(nb: Path) -> bool: """Strip tags from notebook, if there are any. Behavior depends on filename (case-insensitive). - if the notebook file name ends with "_solution_testing", create two files: 1. chop off "_testing" from name, remove test and exercise cells 2. chop off "_solution_testing" from name & add "_exercise", remove solution and test cells - if the file ends with "_testing" only, create one file: 1. chop off "_testing" from name, remove test cells - otherwise: do nothing Returns: Was anything stripped (and new files created)? Raises: IOError: If the desired target for the stripped notebook already exists """ _log.info(f"stripping tags from notebook {nb}") nb_name = nb.stem # Figure out which tags to strip: # - check for case (1), solution_testing.ipynb if nb_name.lower().endswith("_solution_testing"): pre = nb_name[:-17] names = [f"{pre}_solution.ipynb", f"{pre}_exercise.ipynb"] tags_to_strip = [(REMOVE_CELL_TAG, EXERCISE_CELL_TAG), (REMOVE_CELL_TAG, SOLUTION_CELL_TAG)] # - check for case (2), _testing.ipynb elif nb_name.lower().endswith("_testing"): pre = nb_name[:-8] names = [f"{pre}.ipynb"] tags_to_strip = [(REMOVE_CELL_TAG, )] # - if neither, we are done here else: _log.debug(f"notebook '{nb}' does not need to have tags stripped") return False # Create all desired tag-stripped copies for name, remove_tags in zip(names, tags_to_strip): target_nb = nb.parent / name _log.info(f"creating new notebook '{target_nb}'") # Don't overwrite an existing file if target_nb.exists(): _log.warning( f"cannot create new notebook '{target_nb}': file exists") continue # Set up configuration for removing specially tagged cells conf = Config() conf.TagRemovePreprocessor.remove_cell_tags = remove_tags conf.NotebookExporter.preprocessors = [ # this requires the full module path "nbconvert.preprocessors.TagRemovePreprocessor" ] # Convert from Notebook format to Notebook format, stripping tags (body, resources) = NotebookExporter(config=conf).from_filename(str(nb)) # Set up output destination wrt = FilesWriter() wrt.build_directory = str(target_nb.parent) # Write stripped notebook to output file wrt.write(body, resources, notebook_name=target_nb.stem) return True
with open(str(notebook)) as f: nb = nbformat.read(f, as_version=4) ep.preprocess(nb, {'metadata': {'path': str(notebook_pth)}}) print(f"Elapsed time : {time.time()-t}") with open(str(executed_notebook), "w") as f: nbformat.write(nb, f) # 3. Process the notebook we loaded earlier body, resources = html_exporter.from_filename(str(executed_notebook)) writer = FilesWriter() writer.build_directory = str(html_pth) nb_name = executed_notebook.stem.split(".")[0] writer.write(body, resources, notebook_name=nb_name) snippet_map_to_file[nb_name] = Path("notebooks_md") / (str(nb_name)+".md") with open("snippets.yaml", "w") as f: yaml.dump(snippet_map_to_file, f) with open("snippets.md", "w") as f: f.write("# Snippets \n") for key, val in snippet_map_to_file.items(): f.write(f"- [{key.replace('_', ' ')}]({str(val)}) \n")