def convert_notebook(ramp_kit_dir='.'):
import nbformat
import nbconvert
from nbconvert.exporters import HTMLExporter
problem_name = os.path.basename(os.path.abspath(ramp_kit_dir))
print('Testing if the notebook can be converted to html')
notebook_filename = os.path.join(
os.path.abspath(ramp_kit_dir),
'{}_starting_kit.ipynb'.format(problem_name))
notebook_html_filename = os.path.join(
os.path.abspath(ramp_kit_dir),
'{}_starting_kit.html'.format(problem_name))
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
nb_html, _ = nbconvert.export(HTMLExporter, nb)
with open(
os.path.join(os.path.abspath(ramp_kit_dir),
notebook_html_filename), 'wb') as f:
f.write(nb_html.encode('utf-8'))
delete_line_from_file(notebook_html_filename,
'<link rel="stylesheet" href="custom.css">\n')
def notebook_to_md(nbfile, output_dir):
import nbformat, nbconvert, os
from collections import OrderedDict
nb = nbformat.read(nbfile, as_version=nbformat.NO_CONVERT)
dir_name, nb_name = os.path.split(nbfile)
base_nb_name = os.path.splitext(nb_name)[0]
exporter = nbconvert.get_exporter("markdown")
resources = {
'output_files_dir': base_nb_name + "_files",
"build_dir": output_dir
}
mdtext, resources = nbconvert.export(exporter, nb=nb, resources=resources)
# Handle metadata specific to docs.microsoft.com
meta = nb["metadata"].get("ms_docs_meta", {})
docs_meta = OrderedDict()
for key in [
'title', 'description', 'services', 'author', 'manager',
'ms.service', 'ms.technology', 'ms.topic', 'ms.date', 'ms.author'
]:
if key in meta:
docs_meta[key] = meta[key]
if len(docs_meta) > 0:
docs_meta_md = '\n'.join(['---'] +
[': '.join(kv)
for kv in docs_meta.items()] + ['---'])
mdtext = docs_meta_md + '\n' + mdtext
writer = nbconvert.writers.FilesWriter()
writer.write(mdtext, resources, base_nb_name)
def test_markdown_jupytext_nbconvert_is_identity(md_file):
"""Test that a Markdown file, converted to a notebook, then
exported back to Markdown with nbconvert, yields the original file"""
with open(md_file) as fp:
md_org = fp.read()
nb = jupytext.reads(md_org, "md")
import nbconvert
md_nbconvert, _ = nbconvert.export(nbconvert.MarkdownExporter, nb)
# Our expectations
md_expected = md_org.splitlines()
# #region and #endregion comments are removed
md_expected = [
line for line in md_expected
if line not in ["<!-- #region -->", "<!-- #endregion -->"]
]
# language is not inserted by nbconvert
md_expected = [
"```" if line.startswith("```") else line for line in md_expected
]
# nbconvert inserts no empty line after the YAML header (which is in a Raw cell)
md_expected = "\n".join(md_expected).replace("---\n\n", "---\n") + "\n"
# an extra blank line is inserted before code cells
md_nbconvert = md_nbconvert.replace("\n\n```", "\n```")
jupytext.compare.compare(md_nbconvert, md_expected)
def jup2py(jup_filename, py_filename=None, save=True):
# Converting jupyter to py through nbconvertor
py_code = nbconvert.export(nbconvert.PythonExporter(), jup_filename)
mycode = py_code[0].split("\n")
clean_code = []
for index, line in enumerate(mycode):
# Removing which are either "\n" or denotes the "Input" Tag for jupyter notebook
if line == "\n" or (
line.startswith("# In[")) or line is None or line == "":
pass
# Commenting the magic functions or the command line
elif line.startswith("get_ipython()"):
line = "#" + line
clean_code.append(line)
# Remove the jup2py line itself
elif "jup2py" in line:
pass
else:
clean_code.append(line)
clean_code = "\n".join(clean_code)
# Saving back the .py file
if save:
save_pyfile(clean_code, py_filename)
else:
return clean_code
def convert_to_files(names, to_format):
target_dir = os.path.join("static-files", to_format)
for folder in folders:
if not os.path.exists(os.path.join(target_dir, folder)):
os.makedirs(os.path.join(target_dir, folder))
for file_name in names:
p = nbconvert.export(nbconvert.get_exporter(to_format), file_name)
with open(os.path.join(target_dir, file_name[:-6] + p[1]["output_extension"]), 'w') as f:
f.write(p[0].encode("utf-8"))
print file_name
def _reformat_notebook(script):
"""convert a notebook to a script and remove jupyter magics
Parameters
----------
script : str
The path to the Jupyter notebook
Returns
-------
code : str
The notebook as a python script.
"""
nb = nbformat.read(script, nbformat.NO_CONVERT)
code = nbconvert.export(nbconvert.PythonExporter(), nb)[0]
new_code = []
for line in code.split('\n'):
# Exclude magics - HACK - TODO
# Matthew R. Becker 2/27/2017
# So this one is tough. The following magics
#
# %matplotlib inline
# %matplotlib notebook
#
# are only available in jupyter.
# When notebooks are converted to scripts currently (2/17/2017 w/
# version 5.1.1 of nbconvert), these magics are left in the notebook
# as normal ipython magics. However, they cannot be executed by
# ipython and cause an error. There is an open issue in nbconvert to
# exclude these things
# (see https://github.com/jupyter/nbconvert/pull/507). Until this
# issue is addressed, we have to parse them out by hand.
# The block of code here does the following.
# 1) finds jupyter magics at any indentation level (they CAN be
# indented!)
# 2) replaces them with `pass` in python (which does nothing) at the
# same indentation level
if not (line.strip().startswith("get_ipython().magic("
"'matplotlib") or
line.strip().startswith("get_ipython().run_line_magic("
"'matplotlib") or
# python 2 - ugh
line.strip().startswith("get_ipython().magic("
"u'matplotlib") or
line.strip().startswith("get_ipython().run_line_magic("
"u'matplotlib")):
new_code.append(line)
else:
new_code.append(line[:-len(line.lstrip())] + 'pass')
return '\n'.join(new_code)
def convert_to_files(names, to_format):
target_dir = os.path.join("static-files", to_format)
for folder in folders:
if not os.path.exists(os.path.join(target_dir, folder)):
os.makedirs(os.path.join(target_dir, folder))
for file_name in names:
p = nbconvert.export(nbconvert.get_exporter(to_format), file_name)
with open(
os.path.join(target_dir,
file_name[:-6] + p[1]["output_extension"]),
'w') as f:
f.write(p[0].encode("utf-8"))
print file_name
def notebook_to_md(nbfile, output_dir):
import nbformat, nbconvert, os, shutil
from collections import OrderedDict
from base64 import b64encode
nb = nbformat.read(nbfile, as_version=nbformat.NO_CONVERT)
dir_name, nb_name = os.path.split(nbfile)
base_nb_name = os.path.splitext(nb_name)[0]
exporter = nbconvert.get_exporter("markdown")
resources = {
'output_files_dir': base_nb_name + "_files",
"build_dir": output_dir
}
mdtext, resources = nbconvert.export(exporter, nb=nb, resources=resources)
# Handle metadata specific to docs.microsoft.com
meta = nb["metadata"].get("ms_docs_meta", {})
docs_meta = OrderedDict()
for key in [
'title', 'description', 'services', 'author', 'manager',
'ms.service', 'ms.technology', 'ms.topic', 'ms.date', 'ms.author'
]:
if key in meta:
docs_meta[key] = meta[key]
if len(docs_meta) > 0:
docs_meta_md = '\n'.join(['---'] +
[': '.join(kv)
for kv in docs_meta.items()] + ['---'])
mdtext = docs_meta_md + '\n' + mdtext
# Replace images containing paths to file with base64-encoded inline images
for k, v in resources["outputs"].items():
ext = os.path.splitext(k)[-1][1:]
if ext in ("png", "jpg", "svg"):
b64 = b64encode(v)
v2 = "data:{0};base64,{1}".format(ext, b64.decode("ascii"))
mdtext = mdtext.replace(k, v2)
writer = nbconvert.writers.FilesWriter()
writer.write(mdtext, resources, base_nb_name)
shutil.rmtree(resources["output_files_dir"], ignore_errors=True)
return mdtext, resources
def run_test(project, zone, cluster, new_values): # pylint: disable=too-many-locals
# TODO([email protected]): Need to configure the notebook and test to build
# using GCB.
dirname = os.path.dirname(__file__)
if not dirname:
logging.info("__file__ doesn't apper to be absolute path.")
dirname = os.getcwd()
notebook_path = os.path.join(dirname, "TF on GKE.ipynb")
logging.info("Reading notebook %s", notebook_path)
if not os.path.exists(notebook_path):
raise ValueError("%s does not exist" % notebook_path)
with open(notebook_path) as hf:
node = nbformat.read(hf, nbformat.NO_CONVERT)
exporter = nbconvert.PythonExporter()
raw, _ = nbconvert.export(exporter, node)
credentials = GoogleCredentials.get_application_default()
gke = discovery.build("container", "v1", credentials=credentials)
lines = raw.splitlines()
modified = replace_vars(lines, new_values)
modified = strip_appendix(modified)
modified = strip_unexecutable(modified)
with tempfile.NamedTemporaryFile(suffix="notebook.py",
prefix="tmpGke",
mode="w",
delete=False) as hf:
code_path = hf.name
hf.write("\n".join(modified))
logging.info("Wrote notebook to: %s", code_path)
with open(code_path) as hf:
contents = hf.read()
logging.info("Notebook contents:\n%s", contents)
try:
runpy.run_path(code_path)
finally:
logging.info("Deleting cluster; project=%s, zone=%s, name=%s",
project, zone, cluster)
util.delete_cluster(gke, cluster, project, zone)
def test_jupytext_markdown_similar_to_nbconvert(nb_file):
"""Test that the nbconvert export for a notebook matches Jupytext's one"""
nb = jupytext.read(nb_file)
# Remove cell outputs and metadata
for cell in nb.cells:
cell.outputs = []
cell.execution_count = None
cell.metadata = {}
md_jupytext = jupytext.writes(nb, fmt="md")
import nbconvert
md_nbconvert, _ = nbconvert.export(nbconvert.MarkdownExporter, nb)
# our expectations
# nbconvert file has no YAML header
md_jupytext_lines = md_jupytext.splitlines()
_, _, raw_cell, pos = header_to_metadata_and_cell(md_jupytext_lines, "")
md_jupytext = "\n".join(md_jupytext_lines[pos:]) + "\n"
if raw_cell is not None:
md_jupytext = raw_cell.source + "\n\n" + md_jupytext
# region comments are not in nbconvert
md_jupytext = md_jupytext.replace("<!-- #region -->\n",
"").replace("<!-- #endregion -->\n", "")
# Jupytext uses HTML comments to keep track of raw cells
md_jupytext = (md_jupytext.replace("\n<!-- #raw -->\n", "").replace(
"<!-- #raw -->\n", "").replace("\n<!-- #endraw -->\n", ""))
# nbconvert file may start with an empty line
md_jupytext = md_jupytext.lstrip("\n")
md_nbconvert = md_nbconvert.lstrip("\n")
# Jupytext may not always have two blank lines between cells like Jupyter nbconvert
md_jupytext = md_jupytext.replace("\n\n\n", "\n\n")
md_nbconvert = md_nbconvert.replace("\n\n\n", "\n\n")
jupytext.compare.compare(md_nbconvert, md_jupytext)
def _from_ipynb(path_to_nb, exporter, nbconvert_export_kwargs):
"""
Convert ipynb to another format
"""
path = Path(path_to_nb)
nb = nbformat.reads(path.read_text(), as_version=nbformat.NO_CONVERT)
content, _ = nbconvert.export(exporter, nb, **nbconvert_export_kwargs)
if isinstance(content, str):
path.write_text(content)
elif isinstance(content, bytes):
path.write_bytes(content)
else:
raise TypeError('nbconvert returned a converted notebook with'
'unknown format, only text and binary objects '
'are supported')
return content
def test_script(self):
nb_path = TEST_FILES_PATH + "autograder/submission/fails2and6H.ipynb"
nb = nbformat.read(nb_path, as_version=NBFORMAT_VERSION)
os.remove(nb_path)
try:
py, _ = nbconvert.export(nbconvert.PythonExporter, nb)
# remove magic commands
py = "\n".join(l for l in py.split("\n")
if not l.startswith("get_ipython"))
with open(TEST_FILES_PATH + "autograder/submission/fails2and6H.py",
"w+") as f:
f.write(py)
run_autograder(self.config['autograder_dir'])
with open(TEST_FILES_PATH +
"autograder/results/results.json") as f:
actual_results = json.load(f)
self.assertEqual(
actual_results, self.expected_results,
f"Actual results did not matched expected:\n{actual_results}")
self.deletePaths(
[TEST_FILES_PATH + "autograder/submission/fails2and6H.py"])
with open(nb_path, "w+") as f:
nbformat.write(nb, f)
except:
os.chdir(self.cwd)
self.deletePaths(
[TEST_FILES_PATH + "autograder/submission/fails2and6H.py"])
with open(nb_path, "w+") as f:
nbformat.write(nb, f)
raise
def prepare_script(tool_arguments, notebook):
script_lines = []
script_lines.append('from sys import argv')
for index, k in enumerate(tool_arguments):
v = tool_arguments[k]
if isinstance(v, string_types):
line_template = '{argument_name} = argv[{i}]'
else:
line_template = '{argument_name} = {value_type}(argv[{i}])'
script_lines.append(
line_template.format(argument_name=k,
i=index + 1,
value_type=type(v).__name__))
script_lines.extend(
['\n'] +
JINJA2_ENVIRONMENT.get_template('python.jinja2').render().splitlines())
notebook = deepcopy(notebook)
code_cells = get_code_cells(notebook)
code_cells[0]['source'] = '\n'.join(script_lines)
script_content = nbconvert.export(PythonExporter, notebook)[0]
return script_content
def _from_ipynb(path_to_nb, extension, nbconvert_exporter_name):
if nbconvert_exporter_name is not None:
exporter = nbconvert.get_exporter(nbconvert_exporter_name)
else:
try:
exporter = nbconvert.get_exporter(extension.replace('.', ''))
except ValueError:
raise TaskBuildError('Could not determine nbconvert exporter '
'either specify in the path extension '
'or pass a valid exporter name in '
'the NotebookRunner constructor, '
'valid expoers are: {}'.format(
nbconvert.get_export_names()))
path = Path(path_to_nb)
nb = nbformat.v4.reads(path.read_text())
content, _ = nbconvert.export(exporter, nb, exclude_input=True)
path.write_text(content)
return content
def run(
self,
path: str = None,
parameters: dict = None,
output_format: str = None,
exporter_kwargs: dict = None,
) -> str:
"""
Run a Jupyter notebook and output as HTML, notebook, or other formats.
Args:
- path (string, optional): path to fetch the notebook from; can also be
a cloud storage path
- parameters (dict, optional): dictionary of parameters to use for the notebook
- output_format (str, optional): Notebook output format, should be a valid
nbconvert Exporter name. 'json' is treated as 'notebook'.
Valid exporter names: asciidoc, custom, html, latex, markdown,
notebook, pdf, python, rst, script, slides, webpdf. (default: notebook)
- exporter_kwargs (dict, optional): The arguments used for initializing
the exporter.
"""
nb: nbformat.NotebookNode = pm.execute_notebook(
path,
"-",
parameters=parameters,
kernel_name=self.kernel_name,
log_output=self.log_output,
)
if output_format == "json":
output_format = "notebook"
if exporter_kwargs is None:
exporter_kwargs = {}
exporter = nbconvert.get_exporter(output_format)
body, resources = nbconvert.export(exporter, nb, **exporter_kwargs)
return body
""" This script is called by sphinxdoc to convert all Jupyter notebooks in the
Examples folder to ReStructured Text files and stores them in the documentation
folder.
Author: R.A. Collenteur 2016
"""
import nbconvert
from nbconvert import RSTExporter
import glob
import io
import os
notebooks = glob.iglob('../examples/notebooks/*.ipynb')
for nb in notebooks:
x = nbconvert.export(RSTExporter, nb)[0]
with io.open('../doc/examples/%s.rst' % os.path.basename(nb)[:-6],
'w',
encoding='utf-8') as f:
f.write(x[:])
print("example notebooks successfully converted to .rst files")
def run(bytes_in):
myzipfile = zipfile.ZipFile(bytes_in)
with TemporaryDirectory() as temp_dir:
"""Automatically deletes temp_dir if an error occurs"""
print('\nCreate temporary folder: ', temp_dir)
out_formats = ['latex', 'html', 'slides', 'rst', 'markdown']
out_formats_ext = ['tex', 'html', 'slides.html', 'rst', 'md']
# find nam files:
all_nam_files = []
for f in myzipfile.infolist():
if f.is_dir():
# folder
continue
elif f.filename[0] == '_':
# private
continue
elif os.path.basename(f.filename)[0] == '.':
# private
continue
elif f.filename.lower()[-4:] == '.nam':
all_nam_files.append(f)
else:
pass
assert len(
all_nam_files
) != 0, "The input zip does not contain a .nam nor a .NAM file."
assert len(
all_nam_files) == 1, "The zip can only contain a single name file." + \
"The nam files in the zip are: {0}".format(all_nam_files)
# directory name nam file
nam_dir = os.path.join(os.path.dirname(all_nam_files[0].filename), '')
for f in myzipfile.infolist():
if f.is_dir():
# folder
continue
elif f.filename[0] == '_':
# private
continue
elif os.path.basename(f.filename)[0] == '.':
# private
continue
elif f.filename[:len(nam_dir)] == nam_dir:
old_name = f.filename
f.filename = f.filename[len(nam_dir):]
new_name = f.filename
s = 'Copying {0:s} to {1:s}'.format(f.filename, temp_dir)
print(s)
myzipfile.extract(f, temp_dir)
# location to nam file. Always on top of directory
nam_fp = os.path.join(temp_dir,
os.path.basename(all_nam_files[0].filename))
assert os.path.isfile(nam_fp), f'{nam_fp} does not exist'
mp = Model(nam_fp)
nb = mp.script_model2nb(use_yapf=False)
buff = io.BytesIO()
zip_out = zipfile.ZipFile(buff, 'w')
ipynb_buff = io.StringIO(nbformat.writes(nb))
zip_out.writestr('test.ipynb', ipynb_buff.getvalue())
for out_formats_item, out_formats_ext_item in zip(
out_formats, out_formats_ext):
ipynb_buff.seek(0)
zip_out.writestr(
'test.' + out_formats_ext_item,
nbconvert.export(nbconvert.get_exporter(out_formats_item),
ipynb_buff)[0])
ipynb_buff.close()
zip_out.close()
print('\nContent of the zipfile:')
zip_out.printdir()
return buff
try:
from flopymetascript.model import Model
import io
import nbformat
import zipfile
import nbconvert
import flopy
p_list = list(flopy.seawat.Seawat().mfnam_packages.keys())
mp = Model(add_pack=p_list)
# with description
nb = mp.script_model2nb(use_yapf=False)
ipynb_buff = io.StringIO(nbformat.writes(nb))
s = nbconvert.export(nbconvert.get_exporter('markdown'), ipynb_buff)[0]
s = s.replace('\n\n```', '\n\n```python')
p_list_order = list(mp.packages.keys())
toc = [
'* [' + i + '](#' + i.replace('.', '') + ')\n' for i in p_list_order
]
toc = ''.join(toc)
with open('wiki_default_parameters.md', 'w') as f:
f.write(toc)
f.write(s)
ipynb_buff.close()
notebook, pdf, python, rst, script, slides, webpdf. (default: notebook)
- exporter_kwargs (dict, optional): The arguments used for initializing
the exporter.
=======
- output_format (str, optional): Notebook output format.
Currently supported: json, html (default: json)
>>>>>>> prefect clone
"""
nb: nbformat.NotebookNode = pm.execute_notebook(
path, "-", parameters=parameters, kernel_name=self.kernel_name
)
if output_format == "json":
<<<<<<< HEAD
output_format = "notebook"
if exporter_kwargs is None:
exporter_kwargs = {}
exporter = nbconvert.get_exporter(output_format)
body, resources = nbconvert.export(exporter, nb, **exporter_kwargs)
return body
=======
return nbformat.writes(nb)
if output_format == "html":
html_exporter = nbconvert.HTMLExporter()
(body, resources) = html_exporter.from_notebook_node(nb)
return body
raise NotImplementedError("Notebook output %s not supported", output_format)
>>>>>>> prefect clone
def check_files(ipynbfile, pyfile, verbose=True, prefix="FAIL:"):
"""
This method checks a notebook if there are any general errors in it.
After doing this it will take a python file that contains assert statements
and will throw a status code of 2 if there are failures. Will also print
the prefixed error message.
:param ipynbfile: the notebook file
:param pyfile: the python file
:param verbose: turn on verbose mode, will print the code in both files
:param prefix: optional prefix to add to the error message
"""
if os.path.splitext(ipynbfile)[1] != '.ipynb':
raise ValueError(".ipynb file needs to have .ipynb extension")
if os.path.splitext(pyfile)[1] != '.py':
raise ValueError(".py file needs to have .py extension")
with open(ipynbfile, 'r') as readfile:
notebook = nbformat.read(readfile, as_version=nbformat.NO_CONVERT)
output, metadata = nbconvert.export(nbconvert.PythonExporter, notebook)
if verbose:
for i, cell in enumerate(output.split("#")):
code = "#" + cell
click.echo(click.style(f'showing block {i}', fg='blue'))
for line in code.split("\n"):
if line != "\n":
print(" " + line)
try:
exec(code)
except:
click.echo(click.style(f"Error in codeblock!", fg='red'))
traceback.print_exc()
click.echo(
click.style(
f"{prefix}notebook contains error, could not even pass it to py-file.",
fg='red'))
sys.exit(2)
click.echo(click.style(f"Notebook {ipynbfile} has been parsed.",
fg='blue'))
try:
exec(output)
click.echo(
click.style(f"Notebook {ipynbfile} has been executed.", fg='blue'))
except:
click.echo(click.style("ERROR!", fg="red"))
click.echo(
f"{prefix} Jupyter notebook gave an error! Make sure it runs without error."
)
sys.exit(2)
if verbose:
click.echo(click.style(f"Showing contents of {pyfile}.", fg='blue'))
with open(pyfile, "r") as f:
for l in f.readlines():
print(l.replace("\n", ""))
tmpfile = "tmpfile.py"
with open(tmpfile, "a") as f:
f.write(output)
with open(pyfile, "r") as py:
f.write(py.read())
click.echo(
click.style("Temporary file made. About to make test.", fg='blue'))
try:
exec(open(tmpfile).read())
os.remove(tmpfile)
except AssertionError as e:
click.echo(click.style("ASSERTION ERROR DETECTED:", fg='red'))
print(f"{prefix} {e}")
os.remove(tmpfile)
sys.exit(2)
""" This script is called by sphinxdoc to convert all Jupyter notebooks in the
Examples folder to ReStructured Text files and stores them in the documentation
folder.
Author: R.A. Collenteur 2016
"""
import nbconvert
from nbconvert import RSTExporter
import glob
import io
import os
notebooks = glob.iglob('../examples/notebooks/*.ipynb')
for nb in notebooks:
(x,resources) = nbconvert.export(RSTExporter, nb)
with io.open('../doc/examples/%s.rst' % os.path.basename(nb)[:-6], 'w',
encoding='utf-8') as f:
f.write(x[:])
for fname in resources['outputs'].keys():
with io.open('../doc/examples/%s' % fname, 'wb') as f:
f.write(resources['outputs'][fname])
print("example notebooks successfully converted to .rst files")
