def test_tutorial_nb(file_path):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
path of tutorial markdown file
"""
tutorial_name = os.path.basename(file_path)
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
eprocessor = ExecutePreprocessor(timeout=1800)
try:
eprocessor.preprocess(notebook, {'metadata': {}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
finally:
output_nb = open("output.txt", mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
output_nb = open("output.txt", mode='r')
for line in output_nb:
if "Warning:" in line:
fail_dict[tutorial_name] = "%s has warning." % (tutorial_name)
return
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''):
"""
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
"""
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
with io.open(src, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=allow_errors,
timeout=timeout,
kernel_name=kernel_name)
ep.preprocess(nb, resources={})
with io.open(dst, 'wt', encoding='utf-8') as f:
nbformat.write(nb, f)
return dst
def _notebook_run(path):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
kernel_name = "python%d" % sys.version_info[0]
this_file_directory = os.path.dirname(__file__)
errors = []
with tempfile.NamedTemporaryFile(suffix=".ipynb", mode="wt") as fout:
with open(path) as f:
nb = nbformat.read(f, as_version=4)
nb.metadata.get("kernelspec", {})["name"] = kernel_name
ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10)
try:
ep.preprocess(nb, {"metadata": {"path": this_file_directory}})
except CellExecutionError as e:
if "SKIP" in e.traceback:
print(str(e.traceback).split("\n")[-2])
else:
raise e
except TimeoutError as e:
print(e)
finally:
nbformat.write(nb, fout)
# nb = nbformat.read(fout, nbformat.current_nbformat)
# errors = errors.extend(
# [output for cell in nb.cells if "outputs" in cell
# for output in cell["outputs"] if output.output_type == "error"])
return nb, errors
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
"""
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
"""
with io.open(src, encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name)
ep.preprocess(nb, {"metadta": {"path": "notebooks/"}})
with io.open(dst, "wt", encoding="utf-8") as f:
nbformat.write(nb, f)
return dst
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'):
'''
This functions executes a Jupyter notebook and creates the related
HTML file.
Args:
root (str): name of the file without the .ipynb extension
template (str): name of the template (to be in the current folder as template.tpl)
version (int): version of the notebook
timeout (float): maximum time spent per cell
kernel (str)
Returns:
None
The function executes root.ipynb into root_exe.ipynb and creates the file root.html.
'''
with open(root + '.ipynb') as f:
nb = nbformat.read(f, as_version=version)
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
ep.preprocess(nb, {'metadata': {'path': '.'}})
with open(root + '_exe.ipynb', 'wt') as f:
nbformat.write(nb, f)
html_exporter = HTMLExporter()
html_exporter.template_file = template
with open(root + '_exe.ipynb', mode='r') as f:
notebook = nbformat.reads(''.join(f.readlines()), as_version=version)
(body, _) = html_exporter.from_notebook_node(notebook)
codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def run_notebook(notebook_name, nb_kwargs=None,
insert_pos=5, timeout=120, execute_kwargs=None):
"""Runs a notebook and displays the output in the master notebook.
Executes a notebook, optionally passing "arguments" in a way roughly
similar to passing arguments to a function.
Notebook arguments are passed in a dictionary (`nb_kwargs`) which is
converted to a string containing python code, then inserted in the notebook
as a code cell. The code contains only assignments of variables which
can be used to control the execution of a suitably written notebook. When
calling a notebook, you need to know which arguments (variables) to pass.
Differently from functions, no check on the input arguments is performed.
Arguments:
notebook_name (string): name of the notebook to be executed.
nb_kwargs (dict or None): If not None, this dict is converted to a
string of python assignments with keys representing variables
names and values variables content. This string is inserted as
code-cell in the notebook to be executed.
insert_pos (int): position of insertion of the code-cell containing
the input arguments. Default is 5 (i.e. sixth cell). With this
default, the input notebook can define, in the first cell, default
values of input arguments (used when the notebook is executed
with no arguments or through the Notebook GUI).
timeout (int): timeout in seconds after which the execution is aborted.
execute_kwargs (dict): additional arguments passed to
`ExecutePreprocessor`.
"""
if nb_kwargs is not None:
header = '# Cell inserted during automated execution.'
code = dict_to_code(nb_kwargs)
code_cell = '\n'.join((header, code))
if execute_kwargs is None:
execute_kwargs = {}
ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs)
nb = nbformat.read(notebook_name, as_version=4)
if len(nb_kwargs) > 0:
nb['cells'].insert(insert_pos, nbformat.v4.new_code_cell(code_cell))
try:
# Execute the notebook
ep.preprocess(nb, {'metadata': {'path': './'}})
#*****************************************
# Title: analysis.ipynb
# Author: Benjamin RK
# Date: April 2013
# Availability: https://gist.github.com/minrk/5491090
ip = get_ipython()
for cell in nb.cells:
if cell.cell_type != 'code':
continue
ip.run_cell(cell.source)
#*****************************************
except:
# Execution failed, print a message then raise.
msg = 'Error executing the notebook "%s".\n\n' % notebook_name
print(msg)
raise
def _notebook_run(path):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
kernel_name = 'python%d' % sys.version_info[0]
this_file_directory = os.path.dirname(__file__)
errors = []
with tempfile.NamedTemporaryFile(suffix=".ipynb", mode='wt') as fout:
with open(path) as f:
nb = nbformat.read(f, as_version=4)
nb.metadata.get('kernelspec', {})['name'] = kernel_name
ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10)
try:
ep.preprocess(nb, {'metadata': {'path': this_file_directory}})
except CellExecutionError as e:
if "SKIP" in e.traceback:
print(str(e.traceback).split("\n")[-2])
else:
raise e
except RuntimeError as e:
print(e)
finally:
nbformat.write(nb, fout)
return nb, errors
def execute(self):
print("Cleaning lowfat/reports/html ...")
old_reports = os.listdir("lowfat/reports/html")
for old_report in old_reports:
print("- Removing lowfat/reports/html/{}".format(old_report))
os.remove("lowfat/reports/html/{}".format(old_report))
print("Cleaning of lowfat/reports/html is complete.")
notebook_filenames = os.listdir("lowfat/reports")
for notebook_filename in notebook_filenames:
if not notebook_filename.endswith(".ipynb"):
continue
print("Processing lowfat/reports/{}".format(notebook_filename))
# Based on Executing notebooks, nbconvert Documentation by Jupyter Development Team.
# https://nbconvert.readthedocs.io/en/latest/execute_api.html
with open("lowfat/reports/{}".format(notebook_filename)) as file_:
notebook = nbformat.read(file_, as_version=4)
# Kernel is provided by https://github.com/django-extensions/django-extensions/
execute_preprocessor = ExecutePreprocessor(timeout=600, kernel_name='django_extensions')
execute_preprocessor.preprocess(notebook, {'metadata': {'path': '.'}})
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, dummy_resources) = html_exporter.from_notebook_node(notebook)
with open('lowfat/reports/html/{}.html'.format(notebook_filename), 'wt') as file_:
file_.write(body)
def _runTest(self):
kernel = 'python%d' % sys.version_info[0]
cur_dir = os.path.dirname(self.nbfile)
with open(self.nbfile) as f:
nb = nbformat.read(f, as_version=4)
if self.cov:
covdict = {'cell_type': 'code', 'execution_count': 1,
'metadata': {'collapsed': True}, 'outputs': [],
'nbsphinx': 'hidden',
'source': 'import coverage\n'
'coverage.process_startup()\n'
'import sys\n'
'sys.path.append("{0}")\n'.format(cur_dir)
}
nb['cells'].insert(0, nbformat.from_dict(covdict))
exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)
try:
run_dir = os.getenv('WRADLIB_BUILD_DIR', cur_dir)
exproc.preprocess(nb, {'metadata': {'path': run_dir}})
except CellExecutionError as e:
raise e
if self.cov:
nb['cells'].pop(0)
with io.open(self.nbfile, 'wt') as f:
nbformat.write(nb, f)
self.assertTrue(True)
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
'''
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
'''
with io.open(src, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=timeout,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': SOURCE_DIR}})
with io.open(dst, 'wt', encoding='utf-8') as f:
nbformat.write(nb, f)
return dst
def main():
arguments = docopt(__doc__, version='nbgen 2.0')
cmd = subprocess.run([arguments["<path>"]] + arguments["<arguments>"], stdout=subprocess.PIPE)
cmd.check_returncode()
cells = json.loads(cmd.stdout.decode("utf-8"))
nb_dict = {
"metadata": {},
"nbformat": 4,
"nbformat_minor": 0,
"cells": cells,
}
notebook = nbformat.from_dict(nb_dict)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(notebook, {'metadata': {}})
if arguments["nb"]:
nbformat.write(notebook, "{}.ipynb".format(arguments["<name>"]))
elif arguments["slides"]:
config = Config()
reveal_cdn = "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0/"
config.SlidesExporter.reveal_url_prefix = (arguments["--reveal"] or reveal_cdn)
slides, __ = export_slides(nb=notebook, config=config)
with open("{}.html".format(arguments["<name>"]), "w") as html_file:
html_file.write(slides)
def test_notebook(notebook):
nb_name = os.path.split(notebook)[-1]
if nb_name in SLOW_NOTEBOOKS:
pytest.skip('Notebook is too slow to test')
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=240,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def run_ipynb(fs):
"""
"""
with open(fs) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor()
ep.preprocess(nb, {'metadata': {'path': os.path.dirname(fs)}})
with open(fs, 'wt') as f:
nbformat.write(nb, fs)
def write_notebook(cells, outputfile, execute=True, kernel='python3'):
kernelspec = get_kernelspec(kernel)
notebook = new_notebook(cells=cells,
metadata={'language': 'python',
'kernelspec': kernelspec})
if execute:
ep = ExecutePreprocessor(timeout=600, kernelname='python3')
ep.preprocess(notebook,
{'metadata': {'path': os.path.dirname(outputfile)}})
nbformat.write(notebook, outputfile)
def _test_tutorial_nb(tutorial):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
tutorial : str
tutorial name in folder/tutorial format
"""
tutorial_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', '_build', 'html', 'tutorials')
tutorial_path = os.path.join(*([tutorial_dir] + tutorial.split('/')))
# see env variable docs in the doc string of the file
kernel = os.getenv('MXNET_TUTORIAL_TEST_KERNEL', None)
no_cache = os.getenv('MXNET_TUTORIAL_TEST_NO_CACHE', False)
working_dir = os.path.join(*([temp_dir] + tutorial.split('/')))
if no_cache == '1':
print("Cleaning and setting up temp directory '{}'".format(working_dir))
shutil.rmtree(temp_dir, ignore_errors=True)
errors = []
notebook = None
if not os.path.isdir(working_dir):
os.makedirs(working_dir)
try:
notebook = nbformat.read(tutorial_path + '.ipynb', as_version=IPYTHON_VERSION)
# Adding a small delay to allow time for sockets to be freed
# stop-gap measure to battle the 1000ms linger of socket hard coded
# in the kernel API code
time.sleep(1.1)
if kernel is not None:
eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
else:
eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
nb, _ = eprocessor.preprocess(notebook, {'metadata': {'path': working_dir}})
except Exception as err:
err_msg = str(err)
errors.append(err_msg)
finally:
if notebook is not None:
output_file = os.path.join(working_dir, "output.txt")
nbformat.write(notebook, output_file)
output_nb = open(output_file, mode='r')
for line in output_nb:
if "Warning:" in line:
errors.append("Warning:\n"+line)
if len(errors) > 0:
print('\n'.join(errors))
return False
return True
def test_notebooks(tmpdir, filename):
""" Runs a given notebook in a tmpdir """
import pylada
from os.path import join, dirname
from nbformat import read
from nbconvert.preprocessors import ExecutePreprocessor
from sys import version_info
directory = join(dirname(pylada.__file__), 'notebooks')
with open(join(directory, filename + ".ipynb")) as notebook_file:
notebook = read(notebook_file, as_version=4)
preprocessor = ExecutePreprocessor(kernel_name='python%i' %
version_info.major)
preprocessor.preprocess(notebook, {'metadata': {'path': str(tmpdir)}})
def run_notebook(notebook):
# See http://nbconvert.readthedocs.io/en/latest/execute_api.html
# TODO: Specify 'django_extensions' kernel and make it work on the server.
# The kernel can be set as follows:
# ep = ExecutePreprocessor(timeout=120, kernel_name='django_extensions')
# This works locally, but on server, I wasn't able to create the kernel
# (list available kernels by `jupyter kernelspec list`).
# Default kernel currently works, given the `path` (directory from where to
# execute the notebook) is set to //backend. It may fail if some Django
# features are used in the notebook, but I haven't explored this.
ep = ExecutePreprocessor(timeout=120)
path = os.path.join(settings.REPO_DIR, 'backend')
ep.preprocess(notebook, {'metadata': {'path': path}})
def translate(self):
visitor = NBTranslator(self.document, self.app, self.docpath)
self.document.walkabout(visitor)
nb = _finilize_markdown_cells(visitor.nb)
if self.app.config.nbexport_execute:
ep = ExecutePreprocessor(allow_errors=True)
try:
ep.preprocess(nb, {'metadata': {}})
except CellExecutionError as e:
self.app.warn(str(e))
self.output = nbformat.writes(nb)
def test_func(self):
passing = True
print("\n--------------- Testing {0} ---------------".format(nbname))
print(" {0}".format(nbpath))
if nbname in py2Ignore and sys.version_info[0] == 2:
print(" Skipping {}".format(nbname))
return
ep = ClearOutputPreprocessor()
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(
timeout=600,
kernel_name='python{}'.format(sys.version_info[0]),
allow_errors=True
)
out = ex.preprocess(nb, {})
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output['output_type'] == 'error': #in output.keys():
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n <<<<< {} FAILED >>>>> \n
{} in cell [{}] \n-----------\n{}\n-----------\n
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(
nbname, output['ename'],
cell['execution_count'], cell['source'],
err_msg
)
assert passing, msg
print("\n ..... {0} Passed ..... \n".format(nbname))
def notebook_tester(fname):
raw_nb = Exporter().from_filename(fname)
raw_nb[0].metadata.setdefault('kernelspec', {})['name'] = 'python'
preproc = ExecutePreprocessor(timeout=-1)
try:
exec_nb = preproc.preprocess(*raw_nb)
except Exception as e:
return '[Failed]\n{}'.format(e)
out_nb = HTMLExporter().from_notebook_node(*exec_nb)
fout = fname.replace('.ipynb', '.html')
with open(fout, 'w') as f:
f.write(out_nb[0])
return '[Passed]'
def execute_notebook(notebook_path: str):
with open(notebook_path, encoding='utf-8') as notebook:
contents = nbformat.read(notebook, as_version=4)
execution_processor = ExecutePreprocessor(timeout=60, kernel_name="python3")
try:
# Actually execute the notebook in the current working directory.
execution_processor.preprocess(contents, {'metadata': {'path': os.getcwd()}})
return True
except CellExecutionError:
# This is a big chunk of JSON, but the stack trace makes it reasonably
# clear which cell the error occurred in, so fixing it by actually
# running the notebook will probably be easier.
print(contents)
return False
def execute_notebook(npth, dpth, timeout=1200, kernel='python3'):
"""
Execute the notebook at `npth` using `dpth` as the execution
directory. The execution timeout and kernel are `timeout` and
`kernel` respectively.
"""
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
nb = nbformat.read(npth, as_version=4)
t0 = timer()
ep.preprocess(nb, {'metadata': {'path': dpth}})
t1 = timer()
with open(npth, 'wt') as f:
nbformat.write(nb, f)
return t1 - t0
def test_tutorial_notebook():
pytest.importorskip('nbformat')
pytest.importorskip('nbconvert')
pytest.importorskip('matplotlib')
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
rootdir = os.path.join(aospy.__path__[0], 'examples')
with open(os.path.join(rootdir, 'tutorial.ipynb')) as nb_file:
notebook = nbformat.read(nb_file, as_version=nbformat.NO_CONVERT)
kernel_name = 'python' + str(sys.version[0])
ep = ExecutePreprocessor(kernel_name=kernel_name)
with warnings.catch_warnings(record=True):
ep.preprocess(notebook, {})
def run(self):
f = {
'docdir': setup.confdir,
'builddir': setup.app.builder.outdir,
'nbname': self.arguments[0],
}
f['nbpath'] = "{docdir}/../examples/{nbname}.ipynb".format(**f)
f['destdir'] = "{builddir}/examples/{nbname}".format(**f)
if not os.path.exists(f['destdir']):
os.makedirs(f['destdir'])
f['uneval'] = "{destdir}/{nbname}.ipynb".format(**f)
f['eval'] = "{destdir}/{nbname}.eval.ipynb".format(**f)
f['py'] = "{destdir}/{nbname}.py".format(**f)
# 1. Uneval notebook
shutil.copyfile(f['nbpath'], f['uneval'])
with open(f['nbpath']) as nb_f:
nb = nbformat.read(nb_f, as_version=4)
# 2. Python
export_python(nb, f['py'])
# 3. HTML (execute first)
executer = ExecutePreprocessor(timeout=240)
executer.preprocess(nb, {})
html = export_html(nb, f)
# 4. Eval'd notebook
with open(f['eval'], 'w') as eval_f:
nbformat.write(nb, eval_f)
# Create link to notebook and script files
link_rst = "({uneval}; {eval}; {py})".format(
uneval=formatted_link(f['uneval']),
eval=formatted_link(f['eval']),
py=formatted_link(f['py']),
)
rst_file = self.state_machine.document.attributes['source']
self.state_machine.insert_input([link_rst], rst_file)
# create notebook node
nb_node = notebook_node('', html, format='html', source='nb_path')
nb_node.source, nb_node.line = (self.state_machine
.get_source_and_line(self.lineno))
# add dependency
self.state.document.settings.record_dependencies.add(f['nbpath'])
return [nb_node]
def execute_nb(nb):
kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python')
km, kc = start_new_kernel(
kernel_name=kernel_name,
stderr=open(os.devnull, 'w'),
cwd=None)
kc.allow_stdin = False
try:
ep = ExecutePreprocessor(timeout=-1, kernel_name=kernel_name)
nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}})
finally:
kc.stop_channels()
km.shutdown_kernel(now=True)
return nb
def execute_notebook(notebook_filename, notebook_filename_out, params_dict, run_path="", timeout=6000000):
notebook_fp = os.path.join(run_path, notebook_filename)
nb = read_in_notebook(notebook_fp)
new_nb = set_parameters(nb, params_dict)
ep = ExecutePreprocessor(timeout=timeout, kernel_name='python3')
try:
ep.preprocess(new_nb, {'metadata': {'path': run_path}})
except:
msg = 'Error executing the notebook "{0}".\n\n'.format(notebook_filename)
msg = '{0}See notebook "{1}" for the traceback.'.format(msg, notebook_filename_out)
print(msg)
raise
finally:
with open(notebook_filename_out, mode='wt') as f:
nbformat.write(new_nb, f)
def run_notebook(notebook_name, nb_suffix='-out', out_path='.', timeout=3600,
**execute_kwargs):
"""Runs a notebook and saves the output in the same notebook.
Arguments:
notebook_name (string): name of the notebook to be executed.
nb_suffix (string): suffix to append to the file name of the executed
notebook.
timeout (int): timeout in seconds after which the execution is aborted.
execute_kwargs (dict): additional arguments passed to
`ExecutePreprocessor`.
out_path (string): folder where to save the output notebook.
"""
timestamp_cell = "**Executed:** %s\n\n**Duration:** %d seconds."
if str(notebook_name).endswith('.ipynb'):
notebook_name = str(notebook_name)[:-len('.ipynb')]
nb_name_input = notebook_name + '.ipynb'
nb_name_output = notebook_name + '%s.ipynb' % nb_suffix
nb_name_output = os.path.join(out_path, nb_name_output)
print('- Executing: ', nb_name_input)
execute_kwargs_ = dict(kernel_name = 'python%d' % sys.version_info[0])
if execute_kwargs is not None:
execute_kwargs_.update(execute_kwargs)
ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs_)
nb = nbformat.read(nb_name_input, as_version=4)
start_time = time.time()
try:
# Execute the notebook
ep.preprocess(nb, {'metadata': {'path': './'}})
except:
# Execution failed, print a message then raise.
msg = 'Error executing the notebook "%s".\n\n' % notebook_name
msg += 'See notebook "%s" for the traceback.' % nb_name_output
print(msg)
raise
else:
# On successful execution, add timestamping cell
duration = time.time() - start_time
timestamp_cell = timestamp_cell % (time.ctime(start_time), duration)
nb['cells'].insert(0, nbformat.v4.new_markdown_cell(timestamp_cell))
finally:
# Save the notebook even when it raises an error
nbformat.write(nb, nb_name_output)
print('* Output: ', nb_name_output)
def test_tutorial_nb(file_path, workingdir, kernel=None):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
path of tutorial .ipynb file
workingdir: str
path of the directory to run the tutorial in
kernel: str
Default None
name of the kernel to use, if none, will use first kernel
in the list
"""
tutorial_name = os.path.basename(file_path)
sys.stdout.write('Testing {}...'.format(file_path))
sys.stdout.flush()
tick = time.time()
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
if kernel:
eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
else:
eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
success = True
try:
eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
success = False
finally:
output_file = os.path.join(workingdir, "output.txt")
output_nb = open(output_file, mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
output_nb = open(output_file, mode='r')
for line in output_nb:
if "Warning:" in line:
success = False
if tutorial_name in fail_dict:
fail_dict[tutorial_name] += "\n"+line
else:
fail_dict[tutorial_name] = "Warning:\n"+line
sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick ))
sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed'))
sys.stdout.flush()
def nbexec(nb_base, nb):
base_name = nb_base.split('_')[1].split('.')[0].upper()
afp_name = nb.split('_')[1].split('.')[0].upper()
# Lee nb_base origen
with io.open(nb_base, 'rt', encoding="utf8") as f:
aux = nbformat.read(f, as_version=4)
# Reemplaza afp_name
u = aux['cells'][2]['source'].replace(base_name, afp_name)
aux['cells'][2]['source'] = u
# Opciones
ep = ExecutePreprocessor(timeout=300)
# Procesa aux (Revisar si preprocess modifica aux)
ep.preprocess(aux, {})
# Escribe nb destino
with io.open(nb, 'wt', encoding="utf8") as f:
nbformat.write(aux, f)
print "[INFO]--" + datetime.now().strftime('%Y-%M-%d %H:%M:%S') + "--" + "nbexec" + "--" + nb
def execute(self, write=True):
"""
Execute the specified notebook file, and optionally write out the
executed notebook to a new file.
Parameters
----------
write : bool, optional
Write the executed notebook to a new file, or not.
Returns
-------
executed_nb_path : str, ``None``
The path to the executed notebook path, or ``None`` if
``write=False``.
"""
if path.exists(self._executed_nb_path) and not self.overwrite:
logger.debug("Executed notebook already exists at {0}. Use "
"overwrite=True or --overwrite (at cmd line) to re-run"
.format(self._executed_nb_path))
return self._executed_nb_path
# Execute the notebook
logger.debug('Executing notebook using kwargs '
'"{}"...'.format(self._execute_kwargs))
executor = ExecutePreprocessor(**self._execute_kwargs)
with open(self.nb_path) as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
try:
executor.preprocess(nb, {'metadata': {'path': self.path_only}})
except CellExecutionError:
# TODO: should we fail fast and raise, or record all errors?
raise
if write:
logger.debug('Writing executed notebook to file {0}...'
.format(self._executed_nb_path))
with open(self._executed_nb_path, 'w') as f:
nbformat.write(nb, f)
return self._executed_nb_path
def _aux_funct_notebook(self, notebook_filename):
assert os.path.exists(
notebook_filename), f"{notebook_filename} do not exists!"
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
try:
ep = ExecutePreprocessor(timeout=60, store_widget_state=True)
try:
ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_PATHS}})
except CellExecutionError as exc_:
raise
except Exception as exc_:
# error with tqdm progress bar i believe
pass
except CellExecutionError as exc_:
raise
except Exception:
pass
def nbexec(nb_base, nb):
base_name = nb_base.split('_')[1].split('.')[0].upper()
afp_name = nb.split('_')[1].split('.')[0].upper()
# Lee nb_base origen
with io.open(nb_base, 'rt', encoding="utf8") as f:
aux = nbformat.read(f, as_version=4)
# Reemplaza afp_name
u = aux['cells'][2]['source'].replace(base_name, afp_name)
aux['cells'][2]['source'] = u
# Opciones
ep = ExecutePreprocessor(timeout=300)
# Procesa aux (Revisar si preprocess modifica aux)
ep.preprocess(aux, {})
# Escribe nb destino
with io.open(nb, 'wt', encoding="utf8") as f:
nbformat.write(aux, f)
print "[INFO]--" + datetime.now().strftime(
'%Y-%M-%d %H:%M:%S') + "--" + "nbexec" + "--" + nb
def _run_notebook(filename):
# Create the preprocessor.
execute_preprocessor = ExecutePreprocessor(timeout=TIMEOUT,
kernel_name=JUPYTER_KERNEL)
# Open the notebook.
file_path = os.path.dirname(os.path.abspath(filename))
with open(filename) as file_:
notebook = nbformat.read(file_, as_version=4)
with warnings.catch_warnings():
# Silence some spurious warnings.
warnings.filterwarnings('ignore', category=DeprecationWarning)
# Finally, run the notebook.
execute_preprocessor.preprocess(notebook,
{'metadata': {
'path': file_path
}})
def test_reexecute_result_notebook():
with exec_for_test('define_hello_world_pipeline') as result:
assert result.success
materialization_events = [
x for x in result.step_event_list
if x.event_type_value == 'STEP_MATERIALIZATION'
]
for materialization_event in materialization_events:
result_path = get_path(materialization_event)
if result_path.endswith('.ipynb'):
with open(result_path) as fd:
nb = nbformat.read(fd, as_version=4)
ep = ExecutePreprocessor()
ep.preprocess(nb, {})
with open(result_path) as fd:
assert nbformat.read(fd, as_version=4) == nb
def run_notebook(filename):
run_path = os.path.split(filename)[0]
with open(filename) as f:
nb = nbformat.read(f, as_version=4)
try:
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': run_path}})
# FIXME: use tempfile and mv to avoid interruption
# better split the source code of the notebook and the compiled targets.
with open(filename, 'wt') as f:
nbformat.write(nb, f)
except Exception as e:
print('processing', filename, e)
def run(self,
nb_name=None,
cell_timeout=60 * 60,
remove_config=True,
**kwargs):
"""Run the notebook
Paramters
---------
nb_name: str or Path (name of step)
name of the notebook, assumed to be in same directory as step
cell_timeout: int (3600)
number of seconds to allow each cell to execut
remove_config: bool (True)
don't keep serialized version of this class that provides info
to the notebook
"""
# Find path and name of notebook
nb_path = Path(__file__).parent.parent / "steps" / self.step_name
if nb_name is None:
nb_name = self.name
if not nb_name.endswith(".ipynb"):
nb_name = nb_name + ".ipynb"
# Write out pickled self for notebook config
self.args = kwargs
self.interactive = False # to inform notebooks
config_path = nb_path / "config.pkl"
with open(config_path, "wb") as file:
cloudpickle.dump(self, file)
# Run notebook
with open(nb_path / nb_name) as file:
nb = nbformat.read(file, as_version=4)
executer = ExecutePreprocessor(timeout=cell_timeout,
kernel_name="python3")
executer.preprocess(nb, {"metadata": {"path": str(nb_path)}})
# Reload pickled self from config
with open(config_path, "rb") as file:
self = cloudpickle.load(file)
self.manifest.to_csv(self.step_local_staging_dir /
Path("manifest.csv"),
index=False)
if remove_config:
config_path.unlink()
return
def test_radial_fourier_default(hdf5_ds_2, tmpdir_factory, lt_ctx):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'local'}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{
"analysisType": "RADIAL_FOURIER",
"parameters": {
'shape': 'radial_fourier',
'cx': 0,
'cy': 0,
'ri': 0,
'ro': 2,
'n_bins': 2,
'max_order': 23,
}
}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
out = ep.preprocess(nb, {"metadata": {"path": datadir}})
channels = ["dominant_0", "absolute_0_0", "absolute_0_1"]
results = {}
for channel in channels:
data_path = os.path.join(datadir, f"radial_result_{channel}.npy")
results[channel] = np.load(data_path)
analysis = lt_ctx.create_radial_fourier_analysis(dataset=hdf5_ds_2,
cx=0,
cy=0,
ri=0,
ro=2,
n_bins=2,
max_order=23)
expected = lt_ctx.run(analysis)
assert np.allclose(results["dominant_0"], expected["dominant_0"].raw_data)
assert np.allclose(results["absolute_0_0"],
expected["absolute_0_0"].raw_data)
assert np.allclose(results["absolute_0_1"],
expected["absolute_0_1"].raw_data)
def test_com_default(hdf5_ds_2, tmpdir_factory, lt_ctx, local_cluster_url):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{
"analysisType": 'CENTER_OF_MASS',
"parameters": {
'shape': 'com',
'cx': 0,
'cy': 0,
'r': 8,
'ri': 4,
}
}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=30)
ep.preprocess(nb, {"metadata": {"path": datadir}})
channels = ["field", "magnitude", "divergence", "curl", "x", "y"]
results = {}
for channel in channels:
data_path = os.path.join(datadir, f"com_result_{channel}.npy")
results[channel] = np.load(data_path)
com_analysis = lt_ctx.create_com_analysis(
dataset=hdf5_ds_2,
cx=0,
cy=0,
mask_radius=8,
mask_radius_inner=4,
)
expected = lt_ctx.run(com_analysis)
assert np.allclose(results["field"], expected["field"].raw_data)
assert np.allclose(results["magnitude"], expected["magnitude"].raw_data)
assert np.allclose(results["divergence"], expected["divergence"].raw_data)
assert np.allclose(results["curl"], expected["curl"].raw_data)
assert np.allclose(results["x"], expected["x"].raw_data)
assert np.allclose(results["y"], expected["y"].raw_data)
def run_notebook(notebook, root_path=".", timeout=30, prerun=None):
"""Run a notebook in Jupyter
This function will copy all of the files in ``root_path`` to a
temporary directory, run the notebook and then return a
``NotebookResult`` object containing the outputs for each cell.
The notebook is run in a separate process and only objects that
are serializable will be returned in their entirety, otherwise
the string representation will be returned instead.
Parameters
----------
notebook : str
The notebook to run relative to ``root_path``
root_path : str
The root notebook folder (default ".")
timeout : int
Length of time to run the notebook in seconds (default 30)
prerun : function
Function to run prior to starting the notebook, takes the
temporary copy of root_path as a parameter
"""
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
with tempfile.TemporaryDirectory() as td:
workdir = os.path.join(td, 'work')
notebook_dir = os.path.join(workdir, os.path.dirname(notebook))
shutil.copytree(root_path, workdir)
if prerun is not None:
prerun(workdir)
fullpath = os.path.join(workdir, notebook)
with open(fullpath, "r") as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(kernel_name='python3', timeout=timeout)
code_cells = [c for c in nb['cells'] if c['cell_type'] == 'code']
nb['cells'].append(
nbformat.from_dict({'cell_type': 'code',
'metadata': {},
'source': _create_code(len(code_cells))}
))
ep.preprocess(nb, {'metadata': {'path': notebook_dir}})
return NotebookResult(nb)
def run_notebook(self, nb, f):
"""Runs a loaded notebook file."""
if PYTHON_MAJOR_VERSION == 3:
kernel_name = 'python3'
elif PYTHON_MAJOR_VERSION == 2:
kernel_name = 'python2'
else:
raise Exception('Only Python 2 and 3 are supported')
ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name)
try:
ep.preprocess(nb, {'metadata': {'path': '.'}})
except CellExecutionError:
msg = 'Error executing the notebook "%s".\n\n' % f.name
msg += 'See notebook "%s" for the traceback.' % f.name
print(msg)
raise
finally:
nbformat.write(nb, f)
def run_notebook(notebook_path: str) -> Tuple[Any, List[Any]]:
full_path = os.path.join(ROOT_DIR, notebook_path)
nb_name, _ = os.path.splitext(os.path.basename(full_path))
with open(full_path) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=600, kernel_name='python3')
proc.allow_errors = True
proc.preprocess(nb, {'metadata': {'path': os.path.dirname(full_path)}})
errors = []
for cell in nb.cells:
if 'outputs' in cell:
for output in cell['outputs']:
if output.output_type == 'error':
errors.append(output)
return nb, errors
def test_notebook(path):
if path in MARS:
if not os.path.exists(os.path.expanduser("~/.ecmwfapirc")):
pytest.skip("No ~/.ecmwfapirc")
if path in CDS:
if not os.path.exists(os.path.expanduser("~/.cdsapirc")):
pytest.skip("No ~/.cdsapirc")
if path in TENSORFLOW:
if sys.version_info > (3, 8):
pytest.skip("Tensorflow not yet ready on 3.9")
with open(os.path.join(EXAMPLES, path)) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=60 * 60, kernel_name="python3")
proc.preprocess(nb, {"metadata": {"path": EXAMPLES}})
def test_reexecute_result_notebook():
with exec_for_test(
"hello_world_pipeline", {"loggers": {"console": {"config": {"log_level": "ERROR"}}}}
) as result:
assert result.success
materialization_events = [
x for x in result.step_event_list if x.event_type_value == "STEP_MATERIALIZATION"
]
for materialization_event in materialization_events:
result_path = get_path(materialization_event)
if result_path.endswith(".ipynb"):
with open(result_path) as fd:
nb = nbformat.read(fd, as_version=4)
ep = ExecutePreprocessor()
ep.preprocess(nb, {})
with open(result_path) as fd:
assert nbformat.read(fd, as_version=4) == nb
def run_notebook(filename):
execution_failed = False
with open(filename) as f:
nb = nbformat.read(f, as_version=4)
try:
ep = ExecutePreprocessor(timeout=None, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': './'}})
except Exception as e:
print(
"[" + datetime.now().time().strftime('%H:%M') + "] " +
"Error in file '", filename, "': ",
str(e).split('\n')[-2])
execution_failed = True
if not execution_failed:
with open(filename, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
return 1
return 0
def test_run_data_assistants_notebook(tmp_path):
"""
What does this test and why?
One of the resources we have for DataAssistants is a Jupyter notebook that explains/shows the components in code.
This test ensures the codepaths and examples described in the Notebook actually run and pass, nbconvert's
`preprocess` function.
"""
base_dir: str = file_relative_path(
__file__,
"../../../test_fixtures/rule_based_profiler/example_notebooks")
notebook_path: str = os.path.join(
base_dir, "DataAssistants_Instantiation_And_Running.ipynb")
# temporary output notebook for traceback and debugging
output_notebook_path: str = os.path.join(
tmp_path, "DataAssistants_Instantiation_and_running_executed.ipynb")
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=60, kernel_name="python3")
try:
ep.preprocess(nb, {"metadata": {"path": base_dir}})
except CellExecutionError:
msg = 'Error executing the notebook "%s".\n\n' % notebook_path
msg += 'See notebook "%s" for the traceback.' % output_notebook_path
print(msg)
raise
finally:
with open(output_notebook_path, mode="w", encoding="utf-8") as f:
nbformat.write(nb, f)
# clean up Expectations directory after running test
shutil.rmtree(os.path.join(base_dir,
"great_expectations/expectations/tmp"))
os.remove(
os.path.join(base_dir,
"great_expectations/expectations/.ge_store_backend_id"))
os.remove(
os.path.join(base_dir,
"great_expectations/expectations/taxi_data_suite.json"))
def matplotlib_notebook(tmpdir, request):
nb = nbformat.v4.new_notebook()
nb.cells.append(nbformat.v4.new_code_cell('\n'.join(
['import matplotlib.pyplot as plt',
'%matplotlib inline'])))
for _ in range(request.param):
nb.cells.append(nbformat.v4.new_code_cell('\n'.join(
['plt.figure(dpi=100)',
'plt.plot(range(100))',
'plt.show()'])))
ep = ExecutePreprocessor()
ep.preprocess(nb, {'metadata': {'path': tmpdir}})
nb['metadata'].update({'path': f'{tmpdir}'})
return nb
def __init__(self, notebook_filename):
"""
"""
self.outfilename = notebook_filename
# Generate an empty notebook
self.nb = nbf.v4.new_notebook()
# Get all the import from NotebookConfig
self._import()
# Set up some display options for pandas to extend
# the limit of the row and columns that are displayed
self._setup_display()
# Setup matplotlib magic and size of figures
self._setup_matplotlib()
# Load the dataset
self._load_dataset()
# Processor to run the notebook
self.processor = ExecutePreprocessor(timeout=600,
kernel_name='python3',
allow_errors=self.allow_errors)
def execute_notebook(source):
"""
:param source: Jupyter Notebook
:return: Result of the notebook invocation
"""
logger.debug("Executing notebook")
in_memory_source = StringIO(source)
nb = nbformat.read(in_memory_source, as_version=4)
logger.debug("Launching kernels")
ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=True)
ep.preprocess(nb, {'metadata': {'path': '/tmp/'}})
ex = StringIO()
nbformat.write(nb, ex)
logger.debug("Returning results")
return ex.getvalue()
def test_sum_default(hdf5_ds_2, tmpdir_factory):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'local'}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{"analysisType": "SUM_FRAMES", "parameters": {"roi": {}}}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
out = ep.preprocess(nb, {"metadata": {"path": datadir}})
data_path = os.path.join(datadir, 'sum_result.npy')
result = np.load(data_path)
with hdf5_ds_2.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = data.sum(axis=(0, 1))
assert np.allclose(expected, result)
def test_notebook(notebook):
fullfile = os.path.abspath(notebook)
_, filename = os.path.split(fullfile)
filename, _ = os.path.splitext(filename)
if filename in KNOWN_FAILURES:
pytest.skip('{0} is known to fail'.format(filename))
if filename in RPY2_NOTEBOOKS and not HAS_RPY2:
pytest.skip('{0} since rpy2 is not installed'.format(filename))
if filename in JOBLIB_NOTEBOOKS and not JOBLIB_NOTEBOOKS:
pytest.skip('{0} since joblib is not installed'.format(filename))
with io.open(fullfile, encoding='utf-8') as fp:
nb = nbformat.read(fp, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=20,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def run_notebook(notebook_path):
assert os.path.isfile(notebook_path)
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
logging.info("Running: {}".format(notebook_path))
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': 'notebooks/'}})
exec_name = os.path.join(
os.path.dirname(notebook_path),
'_'.join(['executed', os.path.basename(notebook_path)])
)
logging.info("Saving results to: {}".format(exec_name))
with open(exec_name, 'w', encoding='utf-8') as f:
nbformat.write(nb, f)
def run_notebook(self, nb_name):
"""Runs a notebook and write out the output notebook
Args:
nb_name (string): notebook filename
"""
out_notebook_name = 'output_notebooks/%s_result_nb.ipynb' % nb_name
nb = nbformat.read('../data_cleaning/%s' % nb_name, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
try:
ep.preprocess(nb, {'metadata': {'path': '../data_cleaning/'}})
except CellExecutionError:
msg = 'Error executing the notebook'
msg += 'See notebook "%s" for the traceback.' % out_notebook_name
self.logger.info(msg)
raise
finally:
with open(out_notebook_name, mode='wt') as f:
nbformat.write(nb, f)
def setUp(self):
self.path_test_nb = path.join(PATH_FIXTURES, self.name + ".ipynb")
self.path_test_html = path.join(PATH_FIXTURES, self.name + ".html")
self.path_test_js = path.join(PATH_JS_TESTS, self.name + ".js")
self.kernel_name = kernelspec.KERNEL_NAME
with open(self.path_test_nb, "r") as f:
self.nb = nbformat.read(f, as_version=4)
self.ep = ExecutePreprocessor(timeout=600, kernel_name=self.kernel_name)
self.html_exporter = HTMLExporter()
self.ep.preprocess(self.nb, {"metadata": {"path": "."}})
(self.body, _) = self.html_exporter.from_notebook_node(self.nb)
with open(self.path_test_html, "w") as f:
f.write(self.body)
def retrieve_errors(nb_path):
"""
Find all errors that occur when running a nb
"""
with nb_path.open() as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600,
kernel_name='python3',
allow_errors=True)
out, _ = ep.preprocess(nb, {'metadata': {'path': 'nbs/'}})
errors = []
for cell in out.cells:
if "outputs" in cell:
for output in cell["outputs"]:
if output.output_type == "error":
errors.append(output.evalue)
return errors
def test(self):
PYMOO_DIR = get_pymoo()
DOC_DIR = os.path.join(PYMOO_DIR, "doc", "source")
ipynb = []
# collect all the jupyter ipynb in the documentation
for root, directories, filenames in os.walk(DOC_DIR):
for filename in filenames:
if filename.endswith(".ipynb") and "checkpoint" not in filename and not any([filename in s for s in SKIP]):
ipynb.append(os.path.join(root, filename))
i = 0
if STARTING_AT is not None:
for j in range(len(ipynb)):
if STARTING_AT not in ipynb[j]:
i += 1
else:
break
ep = ExecutePreprocessor(timeout=10000, kernel_name='python3')
for i in range(i, len(ipynb)):
fname = ipynb[i]
print(fname.split("/")[-1])
import warnings
warnings.filterwarnings("ignore")
try:
nb = nbformat.read(fname, nbformat.NO_CONVERT)
ep.preprocess(nb, {'metadata': {'path': PYMOO_DIR}})
except CellExecutionError:
msg = 'Error executing the fname "%s".\n\n' % fname
print(msg)
raise
finally:
if OVERWRITE:
with open(fname, mode='wt') as f:
nbformat.write(nb, f)
def run_ipynb(file_path):
"""Execute Jupyter notebook programmatically.
After all cells in the existing notebook have beer run, the function overwrites it.
As a fail-safe, the existing notebook in its pre-run state is saved to
'file_path.parent/.old/'.
Parameters
----------
file_path : str, pathlib.Path
Path to notebook to be run.
References
----------
1. https://nbconvert.readthedocs.io/en/latest/execute_api.html
2. Steven F. Lott, "Modern Python Cookbook", Chapter 9, "Replacing a file while preserving the
previous version", pp. 437-440, 2016.
"""
file_path = Path(file_path)
file_path_temp = file_path.with_suffix('.ipynb.tmp')
file_path.parent.joinpath('.old').mkdir(parents=False, exist_ok=True)
file_path_old = (file_path
.parent
.joinpath('.old', file_path.name)
.with_suffix('.ipynb.old'))
with open(file_path, 'r', encoding='utf-8') as nb:
nbook = nbformat.read(nb, as_version=4)
ep = ExecutePreprocessor(kernel_name='python3')
ep.preprocess(nbook, {'metadata': {'path': file_path.parent}})
with open(file_path_temp, 'w', encoding='utf-8') as nb:
nbformat.write(nbook, nb)
try:
file_path_old.unlink()
except FileNotFoundError:
pass
file_path.rename(file_path_old)
file_path_temp.rename(file_path)
def test_example_notebooks():
""" Tests that all the example ipython notebooks of format
surfinBH/examples/example_*.ipynb are working. Since we expect these to be
used by our users, it would be emabarassing if our own examples failed.
"""
# List of all available fits
fit_names = surfinBH.fits_collection.keys()
for name in fit_names:
short_name = name.split('surfinBH')[-1]
notebook_filename = 'examples/example_%s.ipynb'%short_name
print('testing %s'%notebook_filename)
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=None, kernel_name=python_version)
ep.preprocess(nb, {'metadata': {'path': '.'}})
def execute_notebook(notebook_path: str):
with open(notebook_path, encoding='utf-8') as notebook:
contents = nbformat.read(notebook, as_version=4)
execution_processor = ExecutePreprocessor(timeout=60,
kernel_name="python3")
try:
# Actually execute the notebook in the current working directory.
execution_processor.preprocess(contents,
{'metadata': {
'path': os.getcwd()
}})
return True
except CellExecutionError:
# This is a big chunk of JSON, but the stack trace makes it reasonably
# clear which cell the error occurred in, so fixing it by actually
# running the notebook will probably be easier.
print(contents)
return False
def setUpClass(cls):
print('Running modified notebook...')
notebook_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)), 'demo.ipynb')
kernel_name = "python{0}".format(sys.version_info.major)
with open(notebook_path) as notebook_file:
cls.notebook = nbformat.read(notebook_file,
nbformat.current_nbformat)
cls.notebook = _modify_grid_search_cell(cls.notebook)
ep = ExecutePreprocessor(timeout=600,
kernel_name=kernel_name,
allow_errors=True)
ep.preprocess(
cls.notebook,
{'metadata': {
'path': os.path.dirname(notebook_path)
}})
def execute_notebook(src='estimate.src.ipynb', dest='estimate.ipynb'):
"""Executes the analysis notebook and writes out a copy with all of the
resulting tables and plots.
Parameters
----------
src: str, optional
Source notebook to execute
dest: str, optional
Output notebook
"""
with open(src) as fp:
nb = nbformat.read(fp, 4)
exp = ExecutePreprocessor(timeout=300)
updated_nb, _ = exp.preprocess(nb, {})
with open(dest, 'w') as fp:
nbformat.write(updated_nb, fp)
