def notebook_object_to_rst(ntbk, rpth, cr=None):
"""
Convert notebook object `ntbk` to rst document at `rpth`, in
directory `rdir`. Parameter `cr` is a CrossReferenceLookup
object.
"""
# Parent directory of file rpth
rdir = os.path.dirname(rpth)
# File basename
rb = os.path.basename(os.path.splitext(rpth)[0])
# Pre-process notebook prior to conversion to rst
if cr is not None:
preprocess_notebook(ntbk, cr)
# Convert notebook to rst
rex = RSTExporter()
rsttxt, rstres = rex.from_notebook_node(ntbk)
# Replace `` with ` in sphinx cross-references
rsttxt = re.sub(r':([^:]+):``(.*?)``', r':\1:`\2`', rsttxt)
# Insert a cross-reference target at top of file
reflbl = '.. _examples_' + os.path.basename(rdir) + '_' + \
rb.replace('-', '_') + ':\n'
rsttxt = reflbl + rsttxt
# Write the converted rst to disk
write_notebook_rst(rsttxt, rstres, rb, rdir)
def export_notebook(nb, nb_path, output_dir, SCOPETYPE=None, PLATFORM=None):
"""Takes a notebook node and exports it to ReST and HTML
Args:
nb (notebook): The notebook returned by execute_notebook.
nb_path (str): Path to intput notebook file. Used to generate the
name of the output file.
output_dir (str): The output directory for the ReST and HTML file.
SCOPETYPE (str): Used to generate the output file name.
PLATFORM (str): Used to generate the output file name.
"""
notebook_dir, file_name = os.path.split(nb_path)
file_name_root, _ = os.path.splitext(file_name)
base_path = os.path.join(
output_dir, file_name_root + "-{}-{}".format(SCOPETYPE, PLATFORM))
rst_path = os.path.abspath(base_path + ".rst")
html_path = os.path.abspath(base_path + ".html")
with open(rst_path, "w", encoding='utf-8') as rst_file:
rst_exporter = RSTExporter()
body, res = rst_exporter.from_notebook_node(nb)
rst_file.write(body)
print('Wrote to: ', rst_path)
with open(html_path, 'w', encoding='utf-8') as html_file:
html_exporter = HTMLExporter()
body, res = html_exporter.from_notebook_node(nb)
html_file.write(body)
print('Wrote to: ', html_path)
def notebook_object_to_rst(ntbk, rpth, cr=None):
"""
Convert notebook object `ntbk` to rst document at `rpth`, in
directory `rdir`. Parameter `cr` is a CrossReferenceLookup
object.
"""
# Parent directory of file rpth
rdir = os.path.dirname(rpth)
# File basename
rb = os.path.basename(os.path.splitext(rpth)[0])
# Pre-process notebook prior to conversion to rst
if cr is not None:
preprocess_notebook(ntbk, cr)
# Convert notebook to rst
rex = RSTExporter()
rsttxt, rstres = rex.from_notebook_node(ntbk)
# Replace `` with ` in sphinx cross-references
rsttxt = re.sub(r':([^:]+):``(.*?)``', r':\1:`\2`', rsttxt)
# Insert a cross-reference target at top of file
reflbl = '.. _examples_' + os.path.basename(rdir) + '_' + \
rb.replace('-', '_') + ':\n\n'
rsttxt = reflbl + rsttxt
# Write the converted rst to disk
write_notebook_rst(rsttxt, rstres, rb, rdir)
def nb2rst(filepath):
with open(filepath) as fh:
nb = nbformat.reads(fh.read(), nbformat.NO_CONVERT)
exporter = RSTExporter()
# source is a tuple of python source code
# meta contains metadata
source, meta = exporter.from_notebook_node(nb)
return [line + '\n' for line in source.split('\n')]
def export_notebook(nb, nb_path, output_dir, SCOPETYPE=None, PLATFORM=None):
"""Takes a notebook node and exports it to ReST and HTML
Args:
nb (notebook): The notebook returned by execute_notebook.
nb_path (str): Path to intput notebook file. Used to generate the
name of the output file.
output_dir (str): The output directory for the ReST and HTML file.
SCOPETYPE (str): Used to generate the output file name.
PLATFORM (str): Used to generate the output file name.
"""
notebook_dir, file_name = os.path.split(nb_path)
#need to make sure course is in rst file name
notebook_dir = notebook_dir.replace(r'\\', '_').replace('/', '_')
file_name_root, _ = os.path.splitext(notebook_dir + '_' + file_name)
base_path = os.path.join(
output_dir, file_name_root + '-{}-{}'.format(SCOPETYPE, PLATFORM))
rst_path = os.path.abspath(base_path + '.rst')
html_path = os.path.abspath(base_path + '.html')
# class EscapeBacktickPreprocessor(nbconvert.preprocessors.Preprocessor):
ebp = EscapeBacktickPreprocessor()
rst_ready_nb, _ = ebp.preprocess(nb, {})
with open(rst_path, 'w', encoding='utf-8') as rst_file:
rst_exporter = RSTExporter()
body, res = rst_exporter.from_notebook_node(
rst_ready_nb,
resources={
'unique_key':
'img/{}-{}-{}'.format(SCOPETYPE, PLATFORM,
file_name_root).replace(' ', '')
})
file_names = res['outputs'].keys()
for name in file_names:
with open(os.path.join(output_dir, name), 'wb') as f:
f.write(res['outputs'][name])
print('writing to ', name)
#print(res['outputs'][name])
rst_file.write(body)
print('Wrote to: ', rst_path)
## need resources
with open(html_path, 'w', encoding='utf-8') as html_file:
html_exporter = HTMLExporter()
body, res = html_exporter.from_notebook_node(nb)
html_file.write(body)
print('Wrote to: ', html_path)
def test_codefolding():
""" Test codefolding preprocessor """
nb_name='tests/data/codefolding.ipynb'
with open(nb_name, 'r') as f:
notebook_json = f.read()
notebook = nbformat.reads(notebook_json, as_version=4)
c.RSTExporter.preprocessors = ["pre_codefolding.CodeFoldingPreprocessor"]
c.NbConvertApp.export_format = 'rst'
rst_exporter = RSTExporter(config=c)
body = rst_exporter.from_notebook_node(notebook)
assert 'AXYZ12AXY' not in body[0] # firstline fold
assert 'GR4CX32ZT' not in body[0] # function fold
def test_codefolding():
""" Test codefolding preprocessor """
nb_name = 'tests/data/codefolding.ipynb'
with open(nb_name, 'r') as f:
notebook_json = f.read()
notebook = nbformat.reads(notebook_json, as_version=4)
c.RSTExporter.preprocessors = ["pre_codefolding.CodeFoldingPreprocessor"]
c.NbConvertApp.export_format = 'rst'
rst_exporter = RSTExporter(config=c)
body = rst_exporter.from_notebook_node(notebook)
assert 'AXYZ12AXY' not in body[0] # firstline fold
assert 'GR4CX32ZT' not in body[0] # function fold
def test_pymarkdown_preprocessor():
""" Test python markdown preprocessor """
nb_name = 'tests/data/pymarkdown.ipynb'
with open(nb_name, 'r') as f:
notebook_json = f.read()
notebook = nbformat.reads(notebook_json, as_version=4)
c.RSTExporter.preprocessors = ["pre_pymarkdown.PyMarkdownPreprocessor"]
c.NbConvertApp.export_format = 'rst'
rst_exporter = RSTExporter(config=c)
body = rst_exporter.from_notebook_node(notebook)
with open('test.txt', 'wb') as f:
f.write(body[0].encode('utf8'))
assert 'Hello world' in body[0]
pass
def test_pymarkdown_preprocessor():
""" Test python markdown preprocessor """
nb_name='tests/data/pymarkdown.ipynb'
with open(nb_name, 'r') as f:
notebook_json = f.read()
notebook = nbformat.reads(notebook_json, as_version=4)
c.RSTExporter.preprocessors = ["pre_pymarkdown.PyMarkdownPreprocessor"]
c.NbConvertApp.export_format = 'rst'
rst_exporter = RSTExporter(config=c)
body = rst_exporter.from_notebook_node(notebook)
with open('test.txt', 'wb') as f:
f.write(body[0].encode('utf8'))
assert 'Hello world' in body[0]
pass
def create_notebooks(notebooks):
from nbconvert import RSTExporter
import io
import nbformat
rst_exporter = RSTExporter()
for filename, name in notebooks:
head, tail = os.path.split(filename)
print("\tgenerate {0}.rst".format(name))
with io.open(filename, encoding='utf-8') as source:
notebook = nbformat.reads(source.read(), as_version=4)
(body, resources) = rst_exporter.from_notebook_node(notebook)
body = body.replace(".. code:: python", ".. code-block:: python")
with io.open(filename.replace('.ipynb', '.rst'), "w", encoding='utf-8') as target:
target.write(body)
path = os.path.split(filename)[0]
print(path)
for k, v in resources['outputs'].items():
with io.open(path + '/' + k, "wb") as target:
target.write(v)
class Tester(object):
def __init__(self,path,name):
fname = os.path.join(path, name)
with open(fname, "r") as f:
self.json = f.read()
self.nb = nbformat.reads(self.json, as_version=4)
basename = os.path.splitext(name)[0]
self.outfname = os.path.join(path,'ref',basename + ".new")
self.refname = os.path.join(path,'ref',basename + ".md")
self.exp = RSTExporter()
#self.exp.template_file = 'basic'
def runtest(self):
warnings.filterwarnings("ignore",category=DeprecationWarning)
ep = ExecutePreprocessor(timeout=600, kernel_name='sagemath',allow_errors=True)
ep.preprocess(self.nb, {})
(body, resources) = self.exp.from_notebook_node(self.nb)
with open(self.outfname,'wt') as f:
f.write(body)
if not filecmp.cmp(self.outfname,self.refname):
raise ValueError, "files %s and %s differ" % (self.outfname,self.refname)
def create_notebooks(notebooks):
from nbconvert import RSTExporter
import io
import nbformat
rst_exporter = RSTExporter()
for filename, name in notebooks:
head, tail = os.path.split(filename)
print("\tgenerate {0}.rst".format(name))
with io.open(filename, encoding='utf-8') as source:
notebook = nbformat.reads(source.read(), as_version=4)
(body, resources) = rst_exporter.from_notebook_node(notebook)
body = body.replace(".. code:: python", ".. code-block:: python")
with io.open(filename.replace('.ipynb', '.rst'),
"w",
encoding='utf-8') as target:
target.write(body)
path = os.path.split(filename)[0]
print(path)
for k, v in resources['outputs'].items():
with io.open(path + '/' + k, "wb") as target:
target.write(v)
def convert_to_rst(nb, basedir, fpath, fstem):
# Convert to .rst formats
exp = RSTExporter()
c = Config()
c.TagRemovePreprocessor.remove_cell_tags = {"hide"}
c.TagRemovePreprocessor.remove_input_tags = {"hide-input"}
c.TagRemovePreprocessor.remove_all_outputs_tags = {"hide-output"}
c.ExtractOutputPreprocessor.output_filename_template = (
f"{fstem}_files/{fstem}_" + "{cell_index}_{index}{extension}"
)
exp.register_preprocessor(TagRemovePreprocessor(config=c), True)
exp.register_preprocessor(ExtractOutputPreprocessor(config=c), True)
body, resources = exp.from_notebook_node(nb)
# Clean the output on the notebook and save a .ipynb back to disk
print(f"Writing clean {fpath} ... ", end=" ", flush=True)
nb = strip_output(nb)
with open(fpath, "wt") as f:
nbformat.write(nb, f)
# Write the .rst file
rst_path = os.path.join(basedir, f"{fstem}.rst")
print(f"Writing {rst_path}")
with open(rst_path, "w") as f:
f.write(body)
print(resources["outputs"])
# Write the individual image outputs
imdir = os.path.join(basedir, f"{fstem}_files")
if not os.path.exists(imdir):
os.mkdir(imdir)
for imname, imdata in resources["outputs"].items():
if imname.startswith(fstem):
impath = os.path.join(basedir, f"{imname}")
with open(impath, "wb") as f:
f.write(imdata)
f.write(imdata)
def test_nbconvert_demo(self):
self.maxDiff = None
with open(TESTS_NB_FILE) as f:
notebook = nbformat.read(f, as_version=4)
c = Config()
c.MemestraDeprecationChecker.decorator = ('decoratortest', 'deprecated')
c.RSTExporter.preprocessors = [preprocessor.MemestraDeprecationChecker]
deprecation_checker = RSTExporter(config=c)
# the RST exporter behaves differently on windows and on linux
# there can be some lines with only whitespaces
# so we ignore differences that only consist of empty lines
rst = deprecation_checker.from_notebook_node(notebook)[0]
lines = rst.split('\n')
lines = [l.rstrip() for l in lines]
rst = '\n'.join(lines)
with open(os.path.join(this_dir, 'misc', 'memestra_nb_demo.rst')) as f:
rst_true = f.read()
self.assertEqual(rst, rst_true)
for notebook in notebooks:
try:
with open(notebook, 'rt') as f:
example_nb = f.read()
rst_path = os.path.join(cwd, 'doc', 'source')
path_parts = os.path.split(notebook)
nb_filename = path_parts[-1]
nb_filename = nb_filename.split('.')[0]
source_dir = nb_filename.split('_')[0]
rst_filename = os.path.join(cwd, 'doc', 'source', source_dir,
nb_filename + '.rst')
example_nb = nbformat.reader.reads(example_nb)
rst_export = RSTExporter()
(body, resources) = rst_export.from_notebook_node(example_nb)
with open(rst_filename, 'wt') as rst:
rst.write(body)
for key in resources['outputs'].keys():
if key.endswith('.png'):
resource_filename = os.path.join(cwd, 'doc', 'source',
source_dir, key)
with open(resource_filename, 'wb') as resource:
resource.write(resources['outputs'][key])
except:
import warnings
warnings.warn('Unable to convert {original} to {target}. This '
'only affects documentation generation and not the '
try:
f = open(notebook, 'rt')
example_nb = f.read()
f.close()
rst_path = os.path.join(cwd, 'doc', 'source')
path_parts = os.path.split(notebook)
nb_filename = path_parts[-1]
nb_filename = nb_filename.split('.')[0]
source_dir = nb_filename.split('_')[0]
rst_filename = os.path.join(cwd, 'doc', 'source',
source_dir, nb_filename + '.rst')
example_nb = nbformat.reader.reads(example_nb)
rst_export = RSTExporter()
(body, resources) = rst_export.from_notebook_node(example_nb)
with open(rst_filename, 'wt') as rst:
rst.write(body)
for key in resources['outputs'].keys():
if key.endswith('.png'):
resource_filename = os.path.join(cwd, 'doc', 'source',
source_dir, key)
with open(resource_filename, 'wb') as resource:
resource.write(resources['outputs'][key])
except:
import warnings
warnings.warn('Unable to convert {original} to {target}. This '
'only affects documentation generation and not the '
demo_rst_dir = PathPlus("doc-source/demo_rst").resolve()
if not demo_rst_dir.is_dir():
demo_rst_dir.mkdir()
images_dir = PathPlus("doc-source/examples/graphics").resolve()
if not images_dir.is_dir():
images_dir.mkdir()
for notebook in notebooks:
# Convert the notebook to RST format
markdown_content = PathPlus(f"demo/jupyter/{notebook}.ipynb").read_text()
markdown_content = markdown_content.replace("<!---->", '|')
(body, resources) = rst_exporter.from_notebook_node(
nbformat.reads(markdown_content, as_version=4))
for original, replacement in string_replacements.items():
body = body.replace(original, replacement)
i = 1
while True:
old_body = body
# replace nbconvert syntax with sphinx-toolbox "code-cell" syntax
body = re.sub(r".. code:: ipython\d?",
f".. code-cell:: python\n :execution-count: {i}",
body, 1)
if body == old_body:
break
fields.remove(field)
# Convert to .rst formats
exp = RSTExporter()
c = Config()
c.TagRemovePreprocessor.remove_cell_tags = {"hide"}
c.TagRemovePreprocessor.remove_input_tags = {"hide-input"}
c.TagRemovePreprocessor.remove_all_outputs_tags = {"hide-output"}
c.ExtractOutputPreprocessor.output_filename_template = \
f"{fstem}_files/{fstem}_" + "{cell_index}_{index}{extension}"
exp.register_preprocessor(TagRemovePreprocessor(config=c), True)
exp.register_preprocessor(ExtractOutputPreprocessor(config=c), True)
body, resources = exp.from_notebook_node(nb)
# Clean the output on the notebook and save a .ipynb back to disk
print(f"Writing clean {fpath} ... ", end=" ", flush=True)
nb = strip_output(nb)
with open(fpath, "wt") as f:
nbformat.write(nb, f)
# Write the .rst file
rst_path = os.path.join(basedir, f"{fstem}.rst")
print(f"Writing {rst_path}")
with open(rst_path, "w") as f:
f.write(body)
# Write the individual image outputs
imdir = os.path.join(basedir, f"{fstem}_files")
