def run_nb_test(nb_filename, output_fname):
"""test run"""
print(f'(1) Run notebook: {nb_filename}')
ye_notebook = nbformat.read(nb_filename, as_version=4)
client = NotebookClient(ye_notebook,
timeout=600,
kernel_name='python3',
resources={'metadata': {
'path': '.'
}})
client.execute()
if not os.path.isdir('test_output'):
os.makedirs('test_output')
output_file = f'test_output/{output_fname}'
nbformat.write(ye_notebook, output_file)
print(f'(2) file written: {output_file}')
ye_notebook2 = nbformat.read(output_file, as_version=4)
# 2. Instantiate the exporter. We use the `classic` template for now; we'll get into more details
# later about how to customize the exporter further.
html_exporter = HTMLExporter()
html_exporter.template_name = 'classic'
# 3. Process the notebook we loaded earlier
(body, resources) = html_exporter.from_notebook_node(ye_notebook)
html_filename = f'{output_file}.html'
open(html_filename, 'w').write(body)
print(f'file written: {html_filename}')
def run_notebook(nb_path, tmp_path, env=None, kernel=None, allow_errors=False):
"""Execute a Jupyter notebook via :mod:`nbclient` and collect output.
Parameters
----------
nb_path : path-like
The notebook file to execute.
tmp_path : path-like
A directory in which to create temporary output.
env : dict-like, optional
Execution environment for ``nbconvert``.
If not supplied, :obj:`os.environ` is used.
kernel : str, optional
Jupyter kernel to use. Default: 'python2' or 'python3', matching the
current Python version.
allow_errors : bool, optional
Whether to pass the ``--allow-errors`` option to ``nbconvert``. If
:obj:`True`, the execution always succeeds, and cell output contains
exception information.
Returns
-------
nb : :class:`nbformat.NotebookNode`
Parsed and executed notebook.
errors : list
Any execution errors.
"""
import nbformat
from nbclient import NotebookClient
# Workaround for https://github.com/jupyter/nbclient/issues/85
if (sys.version_info[0] == 3 and sys.version_info[1] >= 8
and sys.platform.startswith("win")):
import asyncio
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
# Read the notebook
with open(nb_path, encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
# Create a client and use it to execute the notebook
client = NotebookClient(nb,
timeout=60,
kernel_name=kernel
or f"python{sys.version_info[0]}",
allow_errors=allow_errors,
resources=dict(metadata=dict(path=tmp_path)))
# Execute the notebook.
# `env` is passed from nbclient to jupyter_client.launcher.launch_kernel()
client.execute(env=env or os.environ.copy())
# Retrieve error information from cells
errors = [
output for cell in nb.cells if "outputs" in cell
for output in cell["outputs"] if output.output_type == "error"
]
return nb, errors
def test_notebook_3d_backend(renderer_notebook, brain_gc):
"""Test executing a notebook that should not fail."""
import nbformat
from nbclient import NotebookClient
notebook_filename = os.path.join(PATH, "test.ipynb")
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
client = NotebookClient(nb)
client.execute()
def main(notebook_paths):
"""
Executes Jupyter Notebooks from the command line.
Expects one or more file paths input as arguments.
Errors are raised and printed to the console.
Example:
$ python run.py ./src/notebooks.ipynb
"""
for path_string in notebook_paths:
click.echo(f"🐍🗒️ {path_string}")
# Get the file name
name = path_string.replace(".ipynb", "")
# Get its parent directory so we can add it to the $PATH
path = pathlib.Path(path_string).parent.absolute()
# Set the intput and output file paths
input_path = f"{name}.ipynb"
output_path = f"{name}-output.ipynb"
# Open up the notebook we're going to run
with open(input_path) as f:
nb = nbformat.read(f, as_version=4)
# Configure nbclient to run the notebook
client = NotebookClient(
# What we want it to run
nb,
timeout=600,
kernel_name='python3',
# We want to allow errors so it doesn't fail silently
allow_errors=False,
force_raise_errors=True,
# Here's where the path gets set
resources={'metadata': {
'path': path
}})
try:
# Run it
client.execute()
except CellExecutionError:
# If there's an error, print it to the terminal.
msg = f"Error executing {input_path}.\n See {output_path} for the traceback."
click.echo(msg)
# And then raise it too
raise
finally:
# Once all that's done, write out the output notebook to the filesystem
with open(output_path, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
def preprocess(self, nb: NotebookNode, resources=None, km=None):
"""
Preprocess notebook executing each code cell.
The input argument *nb* is modified in-place.
Note that this function recalls NotebookClient.__init__, which may look wrong.
However since the preprocess call acts line an init on execution state it's expected.
Therefore, we need to capture it here again to properly reset because traitlet
assignments are not passed. There is a risk if traitlets apply any side effects for
dual init.
The risk should be manageable, and this approach minimizes side-effects relative
to other alternatives.
One alternative but rejected implementation would be to copy the client's init internals
which has already gotten out of sync with nbclient 0.5 release before nbconvert 6.0 released.
Parameters
----------
nb : NotebookNode
Notebook being executed.
resources : dictionary (optional)
Additional resources used in the conversion process. For example,
passing ``{'metadata': {'path': run_path}}`` sets the
execution path to ``run_path``.
km: KernelManager (optional)
Optional kernel manager. If none is provided, a kernel manager will
be created.
Returns
-------
nb : NotebookNode
The executed notebook.
resources : dictionary
Additional resources used in the conversion process.
"""
NotebookClient.__init__(self, nb, km)
self.reset_execution_trackers()
self._check_assign_resources(resources)
with self.setup_kernel():
info_msg = self.wait_for_reply(self.kc.kernel_info())
self.nb.metadata["language_info"] = info_msg["content"][
"language_info"]
for index, cell in enumerate(self.nb.cells):
self.preprocess_cell(cell, resources, index)
self.set_widgets_metadata()
return self.nb, self.resources
def _nbclient():
try:
import nbformat
from jupyter_client import AsyncKernelManager
from nbclient import NotebookClient
from ipywidgets import Button # noqa
import ipyvtklink # noqa
except Exception as exc:
return pytest.skip(f'Skipping Notebook test: {exc}')
km = AsyncKernelManager(config=None)
nb = nbformat.reads("""
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata":{},
"outputs": [],
"source":[]
}
],
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version":3},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}""",
as_version=4)
client = NotebookClient(nb, km=km)
yield client
client._cleanup_kernel()
def _execute_notebook(path):
"""
Private method to execute the provided notebook and handle errors.
"""
with open(path) as f:
nb = nbformat.read(f, as_version=4)
client = NotebookClient(nb,
timeout=600,
kernel_name='python3',
resources={'metadata': {
'path': 'notebooks/'
}})
try:
client.execute()
except CellExecutionError:
out = None
msg = f'Error executing the notebook "{path}".\n\n'
msg += f'See notebook "{path}" for the traceback.'
print(msg)
raise
finally:
with open(path, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
def executeNotebook(nb):
client = NotebookClient(nb, timeout=600, kernel_name='python3')
client.setup_kernel()
with client.setup_kernel():
for index, cell in enumerate(nb.cells):
if cell['cell_type'] == 'code':
try:
result = client.execute_cell(cell, index)
outputs = result['outputs']
except CellExecutionError as e:
pass
# print(e)
client.km.shutdown_kernel()
def autoGradeTestedCells(nb):
client = NotebookClient(nb, timeout=600, kernel_name='python3')
client.setup_kernel()
failedTests = {}
passedAllTests = {}
with client.setup_kernel():
index = 0
while index < len(nb.cells):
cell = nb.cells[index]
directive, source = splitDirective(cell['cell_type'],
cell['source'])
metadata = cell['metadata']
studentID = metadata['studentID'] if ('studentID'
in metadata) else None
isTest = hasDirectiveType(cell, ExamDirectiveType.TEST)
failedTest = False
if cell['cell_type'] == 'code':
try:
result = client.execute_cell(cell, index)
outputs = result['outputs']
except CellExecutionError as e:
if isTest:
failedTest = True
pass
# print(e)
if isTest:
if failedTest:
failedTests[studentID] = True
if studentID in passedAllTests:
del passedAllTests[studentID]
elif not failedTests.get(studentID, False):
passedAllTests[studentID] = True
if hasDirectiveType(cell, ExamDirectiveType.GRADE):
# print('grade', passedAllTests.get(studentID, False), failedTests.get(studentID, False))
if passedAllTests.get(studentID, False):
cell['source'] = cell['source'].replace('TODO', '100')
elif failedTests.get(studentID, False):
cell['source'] = cell['source'].replace('TODO', '0')
elif hasDirectiveType(cell, ExamDirectiveType.COMMENTS):
if passedAllTests.get(studentID, False) or failedTests.get(
studentID, False):
cell['source'] += '\nAutograded'.replace('TODO', '100')
index += 1
client.km.shutdown_kernel()
]
if os.environ.get("PR", "") != "":
notebooks = [
nb for nb in notebooks if not any(avoid in nb for avoid in run_on_master_only)
]
os.chdir("workshop")
for notebook in notebooks:
print(f"Loading {notebook}")
nb = nbformat.read(notebook, as_version=4)
for cell in nb.cells:
if cell.get("cell_type", "") == "code" and "no-execute" in cell.get(
"metadata", {}
).get("tags", []):
cell["cell_type"] = "markdown"
client = NotebookClient(
nb,
timeout=600,
resources={"metadata": {"path": os.path.dirname(notebook)}},
)
try:
client.execute()
nbformat.write(nb, notebook)
except CellExecutionError:
msg = 'Error executing the notebook "%s".\n\n' % notebook
print(msg)
raise
def compile_tutorial(tutorial_name, force_recompile=False):
print('- Tutorial "' + tutorial_name + '"')
notebook_path = 'tutorial_notebooks/' + tutorial_name + '/' + tutorial_name + '.ipynb'
export_path = 'tutorials/' + tutorial_name + '/' + tutorial_name
thumb_dest = os.path.dirname(export_path) + '/thumb.png'
if not os.path.exists(os.path.dirname(export_path)):
os.makedirs(os.path.dirname(export_path))
# Read in notebook
print(' Reading notebook...')
notebook = nbformat.read(notebook_path, 4)
# Scrape title, description and thumbnail
first_cell = notebook.cells[0]
title = first_cell.source.splitlines()[0]
if '#' in title:
title = title.replace('#', '').strip()
description = ''
for line in first_cell.source.splitlines()[1:]:
if line.strip():
description = line.strip()
break
if not description:
print(' Description could not be found in the notebook.')
if 'thumbnail_figure_index' in notebook.metadata:
thumbnail_figure_index = notebook.metadata['thumbnail_figure_index']
else:
thumbnail_figure_index = -1
if 'level' in notebook.metadata:
level = notebook.metadata['level'].capitalize()
elif 'difficulty' in notebook.metadata:
level = notebook.metadata['difficulty'].capitalize()
else:
level = 'Unknown'
# Check if the tutorial was already executed.
if os.path.exists(export_path + '.rst'):
if os.path.getmtime(export_path +
'.rst') > os.path.getmtime(notebook_path):
if force_recompile:
print(' Already compiled. Recompiling anyway...')
else:
print(' Already compiled. Skipping compilation...')
return title, level, description, thumb_dest.split('/', 1)[-1]
# Execute notebook if not already executed
already_executed = any(
c.get('outputs') or c.get('execution_count') for c in notebook.cells
if c.cell_type == 'code')
resources = {'metadata': {'path': os.path.dirname(notebook_path)}}
if not already_executed:
print(' Executing', end='')
start = time.time()
additional_cell_1 = {
"cell_type":
"code",
"execution_count":
None,
"metadata": {},
"outputs": [],
"source":
r"%matplotlib inline" + '\n' +
r"%config InlineBackend.print_figure_kwargs = {'bbox_inches': None}"
}
additional_cell_2 = {
"cell_type":
"code",
"execution_count":
None,
"metadata": {},
"outputs": [],
"source":
"import matplotlib as mpl\nmpl.rcParams['figure.figsize'] = (8, 6)\nmpl.rcParams['figure.dpi'] = 150\nmpl.rcParams['savefig.dpi'] = 150"
}
notebook.cells.insert(1, nbformat.from_dict(additional_cell_1))
notebook.cells.insert(2, nbformat.from_dict(additional_cell_2))
client = NotebookClient(nb=notebook,
resources=resources,
timeout=585,
kernel_name='python3')
try:
with client.setup_kernel():
for i, cell in enumerate(notebook.cells):
print('.', end='')
client.execute_cell(cell, i)
client.set_widgets_metadata()
except CellExecutionError as err:
print(' Error while processing notebook:')
print(' ', err)
print('')
notebook.cells.pop(2)
notebook.cells.pop(1)
end = time.time()
time_taken = end - start
if time_taken > 60:
print(' Execution took %dm%02ds.' %
(time_taken / 60, time_taken % 60))
else:
print(' Execution took %ds.' % time_taken)
else:
print(' Notebook was already executed.')
print(' Rendering tutorial...')
exporter = RSTExporter()
output, resources = exporter.from_notebook_node(notebook, resources)
writer = FilesWriter(build_directory=os.path.dirname(export_path))
writer.write(output,
resources,
notebook_name=os.path.basename(export_path))
pictures = sorted(resources['outputs'], key=output.find)
try:
thumbnail_source = pictures[thumbnail_figure_index]
# Read in thumbnail source image
img = Image.open(os.path.dirname(export_path) + '/' + thumbnail_source)
# Trim whitespace
bg = Image.new(img.mode, img.size, img.getpixel((0, 0)))
diff = ImageChops.difference(img, bg)
diff = ImageChops.add(diff, diff)
bbox = diff.getbbox()
if bbox:
img = img.crop(bbox)
# Resize image to have a width of 400px
img.thumbnail([400, 1000])
# Save thumbnail
img.save(thumb_dest)
except:
shutil.copyfile('_static/no_thumb.png', thumb_dest)
print(' Done!')
return title, level, description, thumb_dest.split('/', 1)[-1]
def __init__(self, **kw):
nb = kw.get('nb')
Preprocessor.__init__(self, nb=nb, **kw)
NotebookClient.__init__(self, nb, **kw)
def notebook_init(nb, resources, allow_errors):
created = False
if not hasattr(notebook_init, "client"):
# create notebook client
client = NotebookClient(nb, resources=resources)
client.allow_errors = allow_errors
client.record_timing = False
client.create_kernel_manager()
client.start_new_kernel()
client.start_new_kernel_client()
info_msg = client.wait_for_reply(client.kc.kernel_info())
client.nb.metadata['language_info'] = info_msg['content'][
'language_info']
notebook_init.client = client
created = True
# cleanup kernel at process exit
atexit.register(client._cleanup_kernel)
else:
# if the kernel has changed we need to force a restart
if nb.metadata.kernelspec.name != notebook_init.client.nb.metadata.kernelspec.name:
raise RestartKernel
# if the input file has changed we need to force a restart
if resources["metadata"]["input"] != notebook_init.client.resources[
"metadata"]["input"]:
raise RestartKernel
# set the new notebook, resources, etc.
notebook_init.client.nb = nb
notebook_init.client.allow_errors = allow_errors
return (notebook_init.client, created)
import nbformat as nbf
from nbclient import NotebookClient
from glob import glob
from pathlib import Path
ROOT = Path(__file__)
DOCS = ROOT.absolute().parent.parent.joinpath("docs")
expected_errors = ["features/notebooks.ipynb"]
notebooks = DOCS.glob("**/*.ipynb")
for path_ntbk in notebooks:
path_ntbk = str(path_ntbk)
print(f"Executing {path_ntbk}")
with open(path_ntbk) as ff:
ntbk = nbf.read(ff, as_version=4)
allow_errors = any(ii in path_ntbk for ii in expected_errors)
client = NotebookClient(ntbk,
allow_errors=allow_errors,
store_widget_state=False)
client.execute()
nbf.write(ntbk, path_ntbk)
