def ipynb2rst(input_fn, output_fn):
with open(input_fn, 'r') as f:
notebook = nbformat.read(f, as_version=4)
for cell in notebook.cells:
if cell.cell_type == 'code':
if '# hide outputs' in cell.source.lower():
cell.outputs = []
if '# hide code' in cell.source.lower():
cell.source = ''
writer = nbconvert.RSTExporter()
sig = hashlib.sha1(input_fn.encode()).hexdigest()[:6]
resources = {
'unique_key':
'output_' + rm_ext(os.path.basename(output_fn)) + '_' + sig
}
(body, resources) = writer.from_notebook_node(notebook, resources)
# Process the raw rst file generated by nbconvert to output a new rst file
body = process_rst(body)
with open(output_fn, 'w') as f:
f.write(body)
outputs = resources['outputs']
base_dir = os.path.dirname(output_fn)
for fn in outputs:
full_fn = os.path.join(base_dir, fn)
with open(full_fn, 'wb') as f:
f.write(outputs[fn])
def to_bitbucket_rst(self):
"""Export the notebook to Bitbucket optimized reST.
- Prepends 'rst-header-' to link targets.
('#some-section' --> '#rst-header-some-section')
"""
check_requirements()
# import on demand to be not required for module import
import nbconvert
from .jinja import FILTERS, LOADER
exporter = nbconvert.RSTExporter()
exporter.filters = FILTERS
exporter.extra_loaders = [LOADER]
# not a real file. will be loaded from string by LOADER
exporter.template_file = 'bitbucket_rst'
rst = exporter.from_filename(self)[0]
# bitbucket_rst template puts code cell input and output
# in single blocks, but can't prevent empty lines in between
#==> Remove them:
return re.sub(
# .. code:: python
#
# >>> input
#
# output
r'(\n >>> .+\n)\s*\n( [^>])',
# .. code:: python
#
# >>> input
# output
r'\1\2',
rst)
def action_nbcompile(cfg, **kwds):
"""Compile notebooks into html.
"""
import nbconvert
src_directory = cfg["notebook"]['src_directory']
len_src_directory = len(src_directory)
nb_filenames = find_notebook_file(src_directory)
# define documentation rst notebook directory
dst_rst_directory = os.path.join("doc", "_notebook")
# remove previous folder
if os.path.exists(dst_rst_directory):
shutil.rmtree(dst_rst_directory)
index_body = """\
========
Notebook
========
.. toctree::
:glob:
:caption: Notebook
"""
rst_exporter = nbconvert.RSTExporter()
for nb_filename in nb_filenames:
# Convert each notebook to rst
body, resources = rst_exporter.from_filename(nb_filename)
# Remove basename src_directory in the path
resources["metadata"]["path"] = resources["metadata"]["path"][len_src_directory + 1:]
# Get the local file_path of the notebook write
local_file_path = os.path.join(resources["metadata"]["path"], resources["metadata"]["name"])
# Add dst_rst_directory in the path
resources["metadata"]["path"] = os.path.join(dst_rst_directory, resources["metadata"]["path"])
# Write beginning block to download file
disclaimer = "This file has been generated from the following notebook: :download:`%s`.\n\n" % os.path.basename(nb_filename)
disclaimer += "Download it if you want to replay it using `jupyter notebook <http://jupyter.org/>`_.\n\n"
body = disclaimer + body
# Write rst with his resources
write_rst_file_with_resources(body, resources)
# Write notebook file for further download
shutil.copy(nb_filename, os.path.join(dst_rst_directory, os.path.basename(nb_filename)))
# Save the notebook rst position in the index body
index_body += " " + local_file_path.replace("\\", "/") + "\n"
# Write rst_index body in the root src directory
rst_index_filename = os.path.join(dst_rst_directory, "index.rst")
write_rst_file_with_filename(index_body, rst_index_filename)
def test_add_html_tab(self):
nb = notebook.split_markdown_cell(notebook.read_markdown(_markdown_src))
nb2 = notebook.get_tab_notebook(nb, tab='python2', default_tab='python3')
nb3 = notebook.get_tab_notebook(nb, tab='python3', default_tab='python3')
nb4 = notebook.merge_tab_notebooks([nb2, nb3])
new_nb = notebook.add_html_tab(nb4, default_tab='python3')
writer = nbconvert.RSTExporter()
body, _ = writer.from_notebook_node(new_nb)
def convert(script, rst, markdown):
"""
Launch Jupyter Notebook (using Docker).
"""
if not script and not rst and not markdown:
all = True
else:
all = False
project_root = get_project_root()
notebooks = os.path.join(project_root, NOTEBOOKS_DIR)
if os.path.isdir(notebooks):
for dirpath, dirnames, filenames in os.walk(notebooks):
for filename in filenames:
if filename.endswith(".ipynb") and os.path.basename(
dirpath) != '.ipynb_checkpoints':
parentpath = os.path.relpath(dirpath, notebooks)
nb = nbf.read(open(os.path.join(dirpath, filename), 'r'),
as_version=4)
if rst or all:
#EXPORT RST
exporter = nbc.RSTExporter()
dir = os.path.join(project_root, RST_DIR, parentpath)
logging.info("Converting the " + filename +
" notebook to an RST format.")
_export(nb, exporter, project_root, dir, parentpath,
filename, '.rst')
if markdown or all:
#EXPORT Markdown
exporter = nbc.MarkdownExporter()
dir = os.path.join(project_root, DOCS_DIR, parentpath)
logging.info("Converting the " + filename +
" notebook to an Markdown format.")
_export(nb, exporter, project_root, dir, parentpath,
filename, '.md')
if script or all:
#EXPORT SCRIPT
exporter = nbc.ScriptExporter()
dir = os.path.join(project_root, SCRIPTS_DIR,
parentpath)
logging.info("Converting the " + filename +
" notebook to a python script.")
_export(nb, exporter, project_root, dir, parentpath,
filename, '.py')
elif filename.endswith(".Rmd"):
#TBD add processing for R notebooks. s
print("R", filename)
def ipynb2rst(input_fn, output_fn):
with open(input_fn, 'r') as f:
notebook = nbformat.read(f, as_version=4)
writer = nbconvert.RSTExporter()
sig = hashlib.sha1(input_fn.encode()).hexdigest()[:6]
resources = {'unique_key':
'output_'+rm_ext(os.path.basename(output_fn))+'_'+sig}
(body, resources) = writer.from_notebook_node(notebook, resources)
body = process_rst(body)
with open(output_fn, 'w') as f:
f.write(body)
outputs = resources['outputs']
base_dir = os.path.dirname(output_fn)
for fn in outputs:
full_fn = os.path.join(base_dir, fn)
with open(full_fn, 'wb') as f:
f.write(outputs[fn])
def run(self):
import nbconvert
from os.path import join
exporter = nbconvert.RSTExporter()
writer = nbconvert.writers.FilesWriter()
files = [
join("examples", "01_simple_usage.ipynb"),
join("examples", "02_advanced_usage.ipynb"),
join("examples", "03_preserving_global_structure.ipynb"),
join("examples", "04_large_data_sets.ipynb"),
]
target_dir = join("docs", "source", "examples")
for fname in files:
self.announce(f"Converting {fname}...")
directory, nb_name = fname.split("/")
nb_name, _ = nb_name.split(".")
body, resources = exporter.from_file(fname)
writer.build_directory = join(target_dir, nb_name)
writer.write(body, resources, nb_name)
def convert_notebook(nb: notebooknode.NotebookNode, resources: Dict[str, str]):
nb = _process_nb(nb)
writer = nbconvert.RSTExporter()
body, resources = writer.from_notebook_node(nb, resources)
body = _process_rst(body)
return body, resources
def create_rst(self, nb, in_dir, odir):
"""Create the rst file from the notebook node"""
exporter = nbconvert.RSTExporter()
raw_rst, resources = exporter.from_notebook_node(nb)
# remove ipython magics
rst_content = ''
i0 = 0
m = None
# HACK: we insert the bokeh style sheets here as well, since for some
# themes (e.g. the sphinx_rtd_theme) it is not sufficient to include
# the style sheets only via app.add_stylesheet
bokeh_str = ''
if 'bokeh' in raw_rst and self.insert_bokeh:
bokeh_str += self.BOKEH_TEMPLATE.format(version=self.insert_bokeh)
if 'bokeh' in raw_rst and self.insert_bokeh_widgets:
bokeh_str += self.BOKEH_WIDGETS_TEMPLATE.format(
version=self.insert_bokeh_widgets)
for m in code_blocks.finditer(raw_rst):
lines = m.group().splitlines(True)
header, content = lines[0], ''.join(lines[1:])
no_magics = magic_patt.sub('\g<1>', content)
# if the code cell only contained magic commands, we skip it
if no_magics.strip():
rst_content += (raw_rst[i0:m.start()] + bokeh_str + header +
no_magics)
bokeh_str = ''
i0 = m.end()
else:
rst_content += raw_rst[i0:m.start()]
i0 = m.end()
if m is not None:
rst_content += bokeh_str + raw_rst[m.end():]
else:
rst_content = raw_rst
rst_content = '.. _%s:\n\n' % self.reference + \
rst_content
language_info = getattr(nb.metadata, 'language_info', {})
url = self.url
if url is not None:
rst_content += self.CODE_DOWNLOAD_NBVIEWER.format(
language=language_info.get('name', 'Python'),
script=os.path.basename(self.script),
nbfile=os.path.basename(self.outfile),
url=url)
else:
rst_content += self.CODE_DOWNLOAD.format(
language=language_info.get('name', 'Python'),
script=os.path.basename(self.script),
nbfile=os.path.basename(self.outfile))
if self.binder_url is not None:
rst_content += self.CODE_RUN_BINDER.format(url=self.binder_url)
supplementary_files = self.supplementary_files
other_supplementary_files = self.other_supplementary_files
if supplementary_files or other_supplementary_files:
for f in (supplementary_files or []) + (other_supplementary_files
or []):
if not os.path.exists(os.path.join(odir, f)):
copyfile(os.path.join(in_dir, f), os.path.join(odir, f))
if supplementary_files:
rst_content += self.data_download(supplementary_files)
rst_file = self.get_out_file()
outputs = sorted(resources['outputs'], key=rst_content.find)
base = os.path.join(
'images',
os.path.splitext(os.path.basename(self.infile))[0] + '_%i.png')
out_map = {
os.path.basename(original): base % i
for i, original in enumerate(outputs)
}
for original, final in six.iteritems(out_map):
rst_content = rst_content.replace(original, final)
with open(rst_file, 'w') \
as f:
f.write(rst_content.rstrip() + '\n')
pictures = []
for original in outputs:
fname = os.path.join(odir, out_map[os.path.basename(original)])
pictures.append(fname)
if six.PY3:
f = open(fname, 'w+b')
else:
f = open(fname, 'w')
f.write(resources['outputs'][original])
f.close()
self.pictures = pictures
def main(cfg, target=".", overwrite=False):
"""Main function called to walk the package
Args:
cfg (Config): current package configuration
target (str): place to write plugin def into
overwrite (bool): whether or not to overwrite previous definition
files. Default to False.
"""
del target
del overwrite
src_directory = cfg["notebook"]['src_directory']
len_src_directory = len(src_directory)
nb_filenames = find_notebook_file(src_directory)
# define documentation rst notebook directory
dst_rst_directory = os.path.join("doc", "_notebook")
# remove previous folder
if os.path.exists(dst_rst_directory):
shutil.rmtree(dst_rst_directory)
index_body = """\
========
Notebook
========
.. toctree::
:glob:
:caption: Notebook
"""
rst_exporter = nbconvert.RSTExporter()
for nb_filename in nb_filenames:
# Convert each notebook to rst
body, resources = rst_exporter.from_filename(nb_filename)
# Remove basename src_directory in the path
resources["metadata"]["path"] = \
resources["metadata"]["path"][len_src_directory + 1:]
# Get the local file_path of the notebook write
local_file_path = os.path.join(resources["metadata"]["path"],
resources["metadata"]["name"])
# Add dst_rst_directory in the path
resources["metadata"]["path"] = os.path.join(
dst_rst_directory, resources["metadata"]["path"])
# Write rst with this resources
write_rst_file_with_resources(body, resources)
# Save the notebook rst position in the index body
index_body += " " + local_file_path.replace("\\", "/") + "\n"
# Write rst_index body in the root src directory
rst_index_filename = os.path.join(dst_rst_directory, "index.rst")
write_rst_file_with_filename(index_body, rst_index_filename)
except:
shutil.rmtree(item)
print("Deleted: {}".format(item))
### RUN ALL NOTEBOOKS
EXEC = not args.noexec
if EXEC:
for f in files:
fp = os.path.join(SOURCE_DIR, f)
if execute_and_save_notebook(fp):
success.append(f)
else:
failed.append(f)
if not args.noconv:
converter = nbconvert.RSTExporter()
writer = nbconvert.writers.FilesWriter()
writer.build_directory = outdir
for f in files:
fp = os.path.join(SOURCE_DIR, f)
try:
resources = init_single_notebook_resources(fp)
(body,
resources) = converter.from_file(fp, resources=resources)
writer.write(body, resources, os.path.splitext(f)[0])
conv_success.append(f)
except Exception as e:
conv_fail.append(f)
