def convert_notebook(nb_path):
prt('Convert notebook to Metatab source package')
if not exists(nb_path):
err("Notebook path does not exist: '{}' ".format(nb_path))
c = Config()
pe = NotebookExecutor(config=c, log=logger)
prt('Running the notebook')
output, resources = pe.from_filename(nb_path)
fw = FilesWriter()
fw.build_directory = pe.output_dir
fw.write(output, resources, notebook_name=DEFAULT_METATAB_FILE)
de = DocumentationExporter(config=c,
log=logger,
metadata=doc_metadata(pe.doc))
prt('Exporting documentation')
output, resources = de.from_filename(nb_path)
fw.build_directory = join(pe.output_dir, 'docs')
fw.write(output, resources, notebook_name='notebook')
new_mt_file = join(pe.output_dir, DEFAULT_METATAB_FILE)
doc = MetapackDoc(new_mt_file)
de.update_metatab(doc, resources)
for lib_dir in pe.lib_dirs:
lib_dir = normpath(lib_dir).lstrip('./')
doc['Resources'].new_term("Root.PythonLib", lib_dir)
path = abspath(lib_dir)
dest = join(pe.output_dir, lib_dir)
ensure_dir(dest)
copytree(path, join(pe.output_dir, lib_dir))
doc.write_csv()
# Reset the input to use the new data
prt('Running with new package file: {}'.format(new_mt_file))
def notebook_to_rst(nbfilename):
nbfilepath = os.path.join(EXPATH, nbfilename)
rstfilename = get_rstfilename(nbfilename)
output_files_dir = only_filename_no_ext(rstfilename)
metadata_path = os.path.dirname(rstfilename)
unique_key = nbfilename.rstrip('.ipynb')
resources = {
'metadata': {'path': metadata_path},
'output_files_dir': output_files_dir,
# Prefix for the output image filenames
'unique_key': unique_key
}
# Read notebook
with open(nbfilepath, 'r') as f:
nb = nbformat.read(f, as_version=4)
# Export
exporter = nbsphinx.Exporter(execute='never', allow_errors=True)
(body, resources) = exporter.from_notebook_node(nb, resources)
# Correct path for the resources
for filename in list(resources['outputs'].keys()):
tmp = os.path.join(RST_PATH, filename)
resources['outputs'][tmp] = resources['outputs'].pop(filename)
fw = FilesWriter()
fw.build_directory = RST_PATH
# Prevent "not in doctree" complains
resources['output_extension'] = ''
body = 'Examples\n--------\n' + body
fw.write(body, resources, notebook_name=rstfilename)
def notebook_to_rst(nbfilename):
nbfilepath = os.path.join(EXPATH, nbfilename)
rstfilename = get_rstfilename(nbfilename)
output_files_dir = only_filename_no_ext(rstfilename)
metadata_path = os.path.dirname(rstfilename)
unique_key = nbfilename.rstrip('.ipynb')
resources = {
'metadata': {'path': metadata_path},
'output_files_dir': output_files_dir,
# Prefix for the output image filenames
'unique_key': unique_key
}
# Read notebook
with open(nbfilepath, 'r') as f:
nb = nbformat.read(f, as_version=4)
# Export
exporter = nbsphinx.Exporter(execute='never', allow_errors=True)
(body, resources) = exporter.from_notebook_node(nb, resources)
# Correct path for the resources
for filename in list(resources['outputs'].keys()):
tmp = os.path.join(RST_PATH, filename)
resources['outputs'][tmp] = resources['outputs'].pop(filename)
fw = FilesWriter()
fw.build_directory = RST_PATH
# Prevent "not in doctree" complains
resources['output_extension'] = ''
body = 'Examples\n--------\n' + body
fw.write(body, resources, notebook_name=rstfilename)
def convert_documentation(nb_path):
"""Run only the document conversion portion of the notebook conversion
The final document will not be completel
"""
with open(nb_path) as f:
nb = nbformat.reads(f.read(), as_version=4)
doc = ExtractInlineMetatabDoc(package_url="metapack+file:" +
dirname(nb_path)).run(nb)
package_name = doc.as_version(None)
output_dir = join(getcwd(), package_name)
de = DocumentationExporter(config=Config(),
log=logger,
metadata=doc_metadata(doc))
prt('Converting documentation')
output, resources = de.from_filename(nb_path)
fw = FilesWriter()
fw.build_directory = join(output_dir, 'docs')
fw.write(output, resources, notebook_name='notebook')
prt("Wrote documentation to {}".format(fw.build_directory))
def _convert(self, tmpdir: Path, entry: Path, outdir: Path, depth: int):
"""Convert a notebook.
Args:
tmpdir: Temporary working directory
entry: notebook to convert
outdir: output directory for .html and .rst files
depth: depth below root, for fixing image paths
"""
test_mode = self.s.get("test_mode")
# strip special cells.
if self._has_tagged_cells(entry, set(self._cell_tags.values())):
_log.debug(f"notebook '{entry.name}' has test cell(s)")
orig_entry, entry = entry, self._strip_tagged_cells(
tmpdir, entry, ("remove", "exercise"), "testing")
notify(f"Stripped tags from: {orig_entry.name}", 3)
else:
# copy to temporary directory just to protect from output cruft
tmp_entry = tmpdir / entry.name
shutil.copy(entry, tmp_entry)
orig_entry, entry = entry, tmp_entry
# convert all tag-stripped versions of the notebook.
# before running, check if converted result is newer than source file
if self._already_converted(orig_entry, entry, outdir):
notify(
f"Skip notebook conversion, output is newer, for: {entry.name}",
3)
self._results.cached.append(entry)
return
notify(f"Running notebook: {entry.name}", 3)
nb = self._parse_and_execute(entry)
if test_mode: # don't do conversion in test mode
return
notify(f"Exporting notebook '{entry.name}' to directory {outdir}", 3)
wrt = FilesWriter()
# export each notebook into multiple target formats
created_wrapper = False
for (exp, postprocess_func, pp_args) in (
(RSTExporter(), self._postprocess_rst, ()),
(HTMLExporter(), self._postprocess_html, (depth, )),
):
_log.debug(
f"export '{orig_entry}' with {exp} to notebook '{entry}'")
(body, resources) = exp.from_notebook_node(nb)
body = postprocess_func(body, *pp_args)
wrt.build_directory = str(outdir)
wrt.write(body, resources, notebook_name=entry.stem)
# create a 'wrapper' page
if not created_wrapper:
_log.debug(
f"create wrapper page for '{entry.name}' in '{outdir}'")
self._create_notebook_wrapper_page(entry.stem, outdir)
created_wrapper = True
# move notebooks into docs directory
_log.debug(f"move notebook '{entry} to output directory: {outdir}")
shutil.copy(entry, outdir / entry.name)
def convert(self, file):
assert os.path.exists(
file), f"this should not happen, path {file} must exist"
body, resources = self.export(file)
fw = FilesWriter()
fw.build_directory = os.path.dirname(file)
f_name = os.path.basename(file).replace(".ipynb", "")
fw.write(body, resources, notebook_name=f_name)
def convert(self, file):
assert os.path.exists(
file), f"this should not happen, path {file} must exist"
body, resources = self.export(file)
fw = FilesWriter()
fw._makedir(self.dst_folder(file))
fw.build_directory = self.dst_folder(file)
fw.write(body,
resources,
notebook_name=self.dest_file(file, withFormat=False))
def export(filename):
c = Config()
# Configure our tag removal
c.TagRemovePreprocessor.remove_cell_tags = ("hide_cell", )
c.TagRemovePreprocessor.remove_all_outputs_tags = ('hide_output', )
c.TagRemovePreprocessor.remove_input_tags = ('hide_input', )
# Configure and run out exporter
c.LatexExporter.preprocessors = [
'nbconvert.preprocessors.TagRemovePreprocessor'
]
targetDirectoryFull_loc = targetDirectoryFull + "\\" + filename.replace(
' ', '_')[:-6]
targetFolder = filename.replace(' ', '_')[:-6]
writer = FilesWriter()
writer.build_directory = targetDirectoryFull_loc
test = writer.write(*CustomLatexExporter(config=c).from_filename(filename),
notebook_name=filename[:-6])
createdFile = targetDirectoryFull_loc + '\\' + filename[:-6] + '.tex'
toBeCreatedFile = targetDirectoryCore + '\\' + filename[:-6] + '.tex'
coreFileName = targetDirectoryCore + '\\' + filename[:-6] + '_Core.tex'
tag = 'maketitle'
lines_to_write = []
tag_found = False
with open(createdFile, encoding="utf8") as in_file:
for line in in_file:
if tag in line:
tag_found = True
elif tag_found:
line = re.sub(r"{In}{incolor}{\d*}", r"{In}{incolor}{In}",
line)
line = re.sub(r"{Out}{outcolor}{\d*}", r"{Out}{outcolor}{Out}",
line)
string = r'JupyterNotebooksFull/' + targetFolder + '/'
line = re.sub(r"(output.*jpg})", string + r"\g<1>", line)
if r'\prompt{Out}{outcolor}{Out}{}' in line:
lines_to_write.append(
r' \begin{tcolorbox}[breakable, size=fbox, boxrule=.5pt, pad at break*=1mm, opacityfill=0]'
+ '\n')
line = r'\prompt{Out}{outcolor}{Out}{\boxspacing}'
lines_to_write.append(line)
if r'{ \hspace*{\fill} \\}' in line:
lines_to_write.append(r'\end{tcolorbox}' + '\n')
lines_to_write = lines_to_write[1:-5]
with open(toBeCreatedFile, 'w+', encoding="utf8") as out_file:
out_file.writelines(lines_to_write)
def _load_documentation_files(self):
from metapack_jupyter.exporters import DocumentationExporter
notebook_docs = []
# First find and remove notebooks from the docs. These wil get processed to create
# normal documents.
try:
for term in list(
self.doc['Documentation'].find('Root.Documentation')):
u = parse_app_url(term.value)
if u is not None and u.target_format == 'ipynb' and u.proto == 'file':
notebook_docs.append(term)
self.doc.remove_term(term)
except KeyError:
self.warn("No documentation defined in metadata")
# Process all of the normal files
super()._load_documentation_files()
fw = FilesWriter()
fw.build_directory = join(self.package_path.path, 'docs')
# Now, generate the notebook documents directly into the filesystem package
for term in notebook_docs:
de = DocumentationExporter(
base_name=term.name or slugify(term.title))
u = parse_app_url(term.value)
nb_path = join(self.source_dir,
u.path) # Only works if the path is relative.
try:
output, resources = de.from_filename(nb_path)
fw.write(output,
resources,
notebook_name=de.base_name +
'_full') # Write notebook html with inputs
de.update_metatab(self.doc, resources)
except Exception as e:
from metapack.cli.core import warn
warn("Failed to convert document for {}: {}".format(
term.name, e))
def convert_to_html(result_notebook: Path) -> Path:
"""
:param result_notebook: The path to the result notebook
:return: Path with output extension
"""
print(f"Running conversion to HTML for {result_notebook}")
with result_notebook.open() as f:
notebook = nbformat.read(f, as_version=4)
html_exporter = HTMLExporter()
html_exporter.exclude_input = True
(body, resources) = html_exporter.from_notebook_node(notebook)
write_file = FilesWriter()
write_file.build_directory = str(result_notebook.parent)
write_file.write(output=body,
resources=resources,
notebook_name=str(result_notebook.stem))
return result_notebook.with_suffix(resources['output_extension'])
def generate_notebook(template_notebook: Path, notebook_params: Dict,
result_notebook: Path) -> Path:
print(f"Writing report to {result_notebook}")
papermill.execute_notebook(input_path=str(template_notebook),
output_path=str(result_notebook),
parameters=notebook_params,
progress_bar=False)
print(f"Running conversion to HTML for {result_notebook}")
with result_notebook.open() as f:
notebook = nbformat.read(f, as_version=4)
html_exporter = HTMLExporter()
html_exporter.exclude_input = True
(body, resources) = html_exporter.from_notebook_node(notebook)
write_file = FilesWriter()
write_file.build_directory = str(result_notebook.parent)
write_file.write(output=body,
resources=resources,
notebook_name=str(result_notebook.stem))
return result_notebook.with_suffix(resources['output_extension'])
def strip_tags(nb: Path) -> bool:
"""Strip tags from notebook, if there are any.
Behavior depends on filename (case-insensitive).
- if the notebook file name ends with "_solution_testing", create two files:
1. chop off "_testing" from name, remove test and exercise cells
2. chop off "_solution_testing" from name & add "_exercise", remove solution and test cells
- if the file ends with "_testing" only, create one file:
1. chop off "_testing" from name, remove test cells
- otherwise: do nothing
Returns:
Was anything stripped (and new files created)?
Raises:
IOError: If the desired target for the stripped notebook already exists
"""
_log.info(f"stripping tags from notebook {nb}")
nb_name = nb.stem
# Figure out which tags to strip:
# - check for case (1), solution_testing.ipynb
if nb_name.lower().endswith("_solution_testing"):
pre = nb_name[:-17]
names = [f"{pre}_solution.ipynb", f"{pre}_exercise.ipynb"]
tags_to_strip = [(REMOVE_CELL_TAG, EXERCISE_CELL_TAG),
(REMOVE_CELL_TAG, SOLUTION_CELL_TAG)]
# - check for case (2), _testing.ipynb
elif nb_name.lower().endswith("_testing"):
pre = nb_name[:-8]
names = [f"{pre}.ipynb"]
tags_to_strip = [(REMOVE_CELL_TAG, )]
# - if neither, we are done here
else:
_log.debug(f"notebook '{nb}' does not need to have tags stripped")
return False
# Create all desired tag-stripped copies
for name, remove_tags in zip(names, tags_to_strip):
target_nb = nb.parent / name
_log.info(f"creating new notebook '{target_nb}'")
# Don't overwrite an existing file
if target_nb.exists():
_log.warning(
f"cannot create new notebook '{target_nb}': file exists")
continue
# Set up configuration for removing specially tagged cells
conf = Config()
conf.TagRemovePreprocessor.remove_cell_tags = remove_tags
conf.NotebookExporter.preprocessors = [
# this requires the full module path
"nbconvert.preprocessors.TagRemovePreprocessor"
]
# Convert from Notebook format to Notebook format, stripping tags
(body,
resources) = NotebookExporter(config=conf).from_filename(str(nb))
# Set up output destination
wrt = FilesWriter()
wrt.build_directory = str(target_nb.parent)
# Write stripped notebook to output file
wrt.write(body, resources, notebook_name=target_nb.stem)
return True
with open(str(notebook)) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {'metadata': {'path': str(notebook_pth)}})
print(f"Elapsed time : {time.time()-t}")
with open(str(executed_notebook), "w") as f:
nbformat.write(nb, f)
# 3. Process the notebook we loaded earlier
body, resources = html_exporter.from_filename(str(executed_notebook))
writer = FilesWriter()
writer.build_directory = str(html_pth)
nb_name = executed_notebook.stem.split(".")[0]
writer.write(body, resources, notebook_name=nb_name)
snippet_map_to_file[nb_name] = Path("notebooks_md") / (str(nb_name)+".md")
with open("snippets.yaml", "w") as f:
yaml.dump(snippet_map_to_file, f)
with open("snippets.md", "w") as f:
f.write("# Snippets \n")
for key, val in snippet_map_to_file.items():
f.write(f"- [{key.replace('_', ' ')}]({str(val)}) \n")
