def test_tutorial_nb(file_path):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
path of tutorial markdown file
"""
tutorial_name = os.path.basename(file_path)
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
eprocessor = ExecutePreprocessor(timeout=1800)
try:
eprocessor.preprocess(notebook, {'metadata': {}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
finally:
output_nb = open("output.txt", mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
output_nb = open("output.txt", mode='r')
for line in output_nb:
if "Warning:" in line:
fail_dict[tutorial_name] = "%s has warning." % (tutorial_name)
return
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''):
"""
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
"""
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
with io.open(src, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=allow_errors,
timeout=timeout,
kernel_name=kernel_name)
ep.preprocess(nb, resources={})
with io.open(dst, 'wt', encoding='utf-8') as f:
nbformat.write(nb, f)
return dst
def _notebook_run(path):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
kernel_name = "python%d" % sys.version_info[0]
this_file_directory = os.path.dirname(__file__)
errors = []
with tempfile.NamedTemporaryFile(suffix=".ipynb", mode="wt") as fout:
with open(path) as f:
nb = nbformat.read(f, as_version=4)
nb.metadata.get("kernelspec", {})["name"] = kernel_name
ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10)
try:
ep.preprocess(nb, {"metadata": {"path": this_file_directory}})
except CellExecutionError as e:
if "SKIP" in e.traceback:
print(str(e.traceback).split("\n")[-2])
else:
raise e
except TimeoutError as e:
print(e)
finally:
nbformat.write(nb, fout)
# nb = nbformat.read(fout, nbformat.current_nbformat)
# errors = errors.extend(
# [output for cell in nb.cells if "outputs" in cell
# for output in cell["outputs"] if output.output_type == "error"])
return nb, errors
def run_notebook(notebook_name, nb_kwargs=None,
insert_pos=5, timeout=120, execute_kwargs=None):
"""Runs a notebook and displays the output in the master notebook.
Executes a notebook, optionally passing "arguments" in a way roughly
similar to passing arguments to a function.
Notebook arguments are passed in a dictionary (`nb_kwargs`) which is
converted to a string containing python code, then inserted in the notebook
as a code cell. The code contains only assignments of variables which
can be used to control the execution of a suitably written notebook. When
calling a notebook, you need to know which arguments (variables) to pass.
Differently from functions, no check on the input arguments is performed.
Arguments:
notebook_name (string): name of the notebook to be executed.
nb_kwargs (dict or None): If not None, this dict is converted to a
string of python assignments with keys representing variables
names and values variables content. This string is inserted as
code-cell in the notebook to be executed.
insert_pos (int): position of insertion of the code-cell containing
the input arguments. Default is 5 (i.e. sixth cell). With this
default, the input notebook can define, in the first cell, default
values of input arguments (used when the notebook is executed
with no arguments or through the Notebook GUI).
timeout (int): timeout in seconds after which the execution is aborted.
execute_kwargs (dict): additional arguments passed to
`ExecutePreprocessor`.
"""
if nb_kwargs is not None:
header = '# Cell inserted during automated execution.'
code = dict_to_code(nb_kwargs)
code_cell = '\n'.join((header, code))
if execute_kwargs is None:
execute_kwargs = {}
ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs)
nb = nbformat.read(notebook_name, as_version=4)
if len(nb_kwargs) > 0:
nb['cells'].insert(insert_pos, nbformat.v4.new_code_cell(code_cell))
try:
# Execute the notebook
ep.preprocess(nb, {'metadata': {'path': './'}})
#*****************************************
# Title: analysis.ipynb
# Author: Benjamin RK
# Date: April 2013
# Availability: https://gist.github.com/minrk/5491090
ip = get_ipython()
for cell in nb.cells:
if cell.cell_type != 'code':
continue
ip.run_cell(cell.source)
#*****************************************
except:
# Execution failed, print a message then raise.
msg = 'Error executing the notebook "%s".\n\n' % notebook_name
print(msg)
raise
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
'''
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
'''
with io.open(src, encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=timeout,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': SOURCE_DIR}})
with io.open(dst, 'wt', encoding='utf-8') as f:
nbformat.write(nb, f)
return dst
def execute(self):
print("Cleaning lowfat/reports/html ...")
old_reports = os.listdir("lowfat/reports/html")
for old_report in old_reports:
print("- Removing lowfat/reports/html/{}".format(old_report))
os.remove("lowfat/reports/html/{}".format(old_report))
print("Cleaning of lowfat/reports/html is complete.")
notebook_filenames = os.listdir("lowfat/reports")
for notebook_filename in notebook_filenames:
if not notebook_filename.endswith(".ipynb"):
continue
print("Processing lowfat/reports/{}".format(notebook_filename))
# Based on Executing notebooks, nbconvert Documentation by Jupyter Development Team.
# https://nbconvert.readthedocs.io/en/latest/execute_api.html
with open("lowfat/reports/{}".format(notebook_filename)) as file_:
notebook = nbformat.read(file_, as_version=4)
# Kernel is provided by https://github.com/django-extensions/django-extensions/
execute_preprocessor = ExecutePreprocessor(timeout=600, kernel_name='django_extensions')
execute_preprocessor.preprocess(notebook, {'metadata': {'path': '.'}})
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
(body, dummy_resources) = html_exporter.from_notebook_node(notebook)
with open('lowfat/reports/html/{}.html'.format(notebook_filename), 'wt') as file_:
file_.write(body)
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'):
'''
This functions executes a Jupyter notebook and creates the related
HTML file.
Args:
root (str): name of the file without the .ipynb extension
template (str): name of the template (to be in the current folder as template.tpl)
version (int): version of the notebook
timeout (float): maximum time spent per cell
kernel (str)
Returns:
None
The function executes root.ipynb into root_exe.ipynb and creates the file root.html.
'''
with open(root + '.ipynb') as f:
nb = nbformat.read(f, as_version=version)
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
ep.preprocess(nb, {'metadata': {'path': '.'}})
with open(root + '_exe.ipynb', 'wt') as f:
nbformat.write(nb, f)
html_exporter = HTMLExporter()
html_exporter.template_file = template
with open(root + '_exe.ipynb', mode='r') as f:
notebook = nbformat.reads(''.join(f.readlines()), as_version=version)
(body, _) = html_exporter.from_notebook_node(notebook)
codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def main():
arguments = docopt(__doc__, version='nbgen 2.0')
cmd = subprocess.run([arguments["<path>"]] + arguments["<arguments>"], stdout=subprocess.PIPE)
cmd.check_returncode()
cells = json.loads(cmd.stdout.decode("utf-8"))
nb_dict = {
"metadata": {},
"nbformat": 4,
"nbformat_minor": 0,
"cells": cells,
}
notebook = nbformat.from_dict(nb_dict)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(notebook, {'metadata': {}})
if arguments["nb"]:
nbformat.write(notebook, "{}.ipynb".format(arguments["<name>"]))
elif arguments["slides"]:
config = Config()
reveal_cdn = "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0/"
config.SlidesExporter.reveal_url_prefix = (arguments["--reveal"] or reveal_cdn)
slides, __ = export_slides(nb=notebook, config=config)
with open("{}.html".format(arguments["<name>"]), "w") as html_file:
html_file.write(slides)
def _runTest(self):
kernel = 'python%d' % sys.version_info[0]
cur_dir = os.path.dirname(self.nbfile)
with open(self.nbfile) as f:
nb = nbformat.read(f, as_version=4)
if self.cov:
covdict = {'cell_type': 'code', 'execution_count': 1,
'metadata': {'collapsed': True}, 'outputs': [],
'nbsphinx': 'hidden',
'source': 'import coverage\n'
'coverage.process_startup()\n'
'import sys\n'
'sys.path.append("{0}")\n'.format(cur_dir)
}
nb['cells'].insert(0, nbformat.from_dict(covdict))
exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600)
try:
run_dir = os.getenv('WRADLIB_BUILD_DIR', cur_dir)
exproc.preprocess(nb, {'metadata': {'path': run_dir}})
except CellExecutionError as e:
raise e
if self.cov:
nb['cells'].pop(0)
with io.open(self.nbfile, 'wt') as f:
nbformat.write(nb, f)
self.assertTrue(True)
def _notebook_run(path):
"""Execute a notebook via nbconvert and collect output.
:returns (parsed nb object, execution errors)
"""
kernel_name = 'python%d' % sys.version_info[0]
this_file_directory = os.path.dirname(__file__)
errors = []
with tempfile.NamedTemporaryFile(suffix=".ipynb", mode='wt') as fout:
with open(path) as f:
nb = nbformat.read(f, as_version=4)
nb.metadata.get('kernelspec', {})['name'] = kernel_name
ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10)
try:
ep.preprocess(nb, {'metadata': {'path': this_file_directory}})
except CellExecutionError as e:
if "SKIP" in e.traceback:
print(str(e.traceback).split("\n")[-2])
else:
raise e
except RuntimeError as e:
print(e)
finally:
nbformat.write(nb, fout)
return nb, errors
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None):
"""
Execute notebook in `src` and write the output to `dst`
Parameters
----------
src, dst: str
path to notebook
allow_errors: bool
timeout: int
kernel_name: str
defualts to value set in notebook metadata
Returns
-------
dst: str
"""
with io.open(src, encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name)
ep.preprocess(nb, {"metadta": {"path": "notebooks/"}})
with io.open(dst, "wt", encoding="utf-8") as f:
nbformat.write(nb, f)
return dst
def test_notebook(notebook):
nb_name = os.path.split(notebook)[-1]
if nb_name in SLOW_NOTEBOOKS:
pytest.skip('Notebook is too slow to test')
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=240,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def run_ipynb(fs):
"""
"""
with open(fs) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor()
ep.preprocess(nb, {'metadata': {'path': os.path.dirname(fs)}})
with open(fs, 'wt') as f:
nbformat.write(nb, fs)
def write_notebook(cells, outputfile, execute=True, kernel='python3'):
kernelspec = get_kernelspec(kernel)
notebook = new_notebook(cells=cells,
metadata={'language': 'python',
'kernelspec': kernelspec})
if execute:
ep = ExecutePreprocessor(timeout=600, kernelname='python3')
ep.preprocess(notebook,
{'metadata': {'path': os.path.dirname(outputfile)}})
nbformat.write(notebook, outputfile)
def test_notebooks(tmpdir, filename):
""" Runs a given notebook in a tmpdir """
import pylada
from os.path import join, dirname
from nbformat import read
from nbconvert.preprocessors import ExecutePreprocessor
from sys import version_info
directory = join(dirname(pylada.__file__), 'notebooks')
with open(join(directory, filename + ".ipynb")) as notebook_file:
notebook = read(notebook_file, as_version=4)
preprocessor = ExecutePreprocessor(kernel_name='python%i' %
version_info.major)
preprocessor.preprocess(notebook, {'metadata': {'path': str(tmpdir)}})
def run_notebook(notebook):
# See http://nbconvert.readthedocs.io/en/latest/execute_api.html
# TODO: Specify 'django_extensions' kernel and make it work on the server.
# The kernel can be set as follows:
# ep = ExecutePreprocessor(timeout=120, kernel_name='django_extensions')
# This works locally, but on server, I wasn't able to create the kernel
# (list available kernels by `jupyter kernelspec list`).
# Default kernel currently works, given the `path` (directory from where to
# execute the notebook) is set to //backend. It may fail if some Django
# features are used in the notebook, but I haven't explored this.
ep = ExecutePreprocessor(timeout=120)
path = os.path.join(settings.REPO_DIR, 'backend')
ep.preprocess(notebook, {'metadata': {'path': path}})
def translate(self):
visitor = NBTranslator(self.document, self.app, self.docpath)
self.document.walkabout(visitor)
nb = _finilize_markdown_cells(visitor.nb)
if self.app.config.nbexport_execute:
ep = ExecutePreprocessor(allow_errors=True)
try:
ep.preprocess(nb, {'metadata': {}})
except CellExecutionError as e:
self.app.warn(str(e))
self.output = nbformat.writes(nb)
def execute_notebook(notebook_path: str):
with open(notebook_path, encoding='utf-8') as notebook:
contents = nbformat.read(notebook, as_version=4)
execution_processor = ExecutePreprocessor(timeout=60, kernel_name="python3")
try:
# Actually execute the notebook in the current working directory.
execution_processor.preprocess(contents, {'metadata': {'path': os.getcwd()}})
return True
except CellExecutionError:
# This is a big chunk of JSON, but the stack trace makes it reasonably
# clear which cell the error occurred in, so fixing it by actually
# running the notebook will probably be easier.
print(contents)
return False
def execute_notebook(npth, dpth, timeout=1200, kernel='python3'):
"""
Execute the notebook at `npth` using `dpth` as the execution
directory. The execution timeout and kernel are `timeout` and
`kernel` respectively.
"""
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
nb = nbformat.read(npth, as_version=4)
t0 = timer()
ep.preprocess(nb, {'metadata': {'path': dpth}})
t1 = timer()
with open(npth, 'wt') as f:
nbformat.write(nb, f)
return t1 - t0
def test_tutorial_notebook():
pytest.importorskip('nbformat')
pytest.importorskip('nbconvert')
pytest.importorskip('matplotlib')
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
rootdir = os.path.join(aospy.__path__[0], 'examples')
with open(os.path.join(rootdir, 'tutorial.ipynb')) as nb_file:
notebook = nbformat.read(nb_file, as_version=nbformat.NO_CONVERT)
kernel_name = 'python' + str(sys.version[0])
ep = ExecutePreprocessor(kernel_name=kernel_name)
with warnings.catch_warnings(record=True):
ep.preprocess(notebook, {})
def run(self):
f = {
'docdir': setup.confdir,
'builddir': setup.app.builder.outdir,
'nbname': self.arguments[0],
}
f['nbpath'] = "{docdir}/../examples/{nbname}.ipynb".format(**f)
f['destdir'] = "{builddir}/examples/{nbname}".format(**f)
if not os.path.exists(f['destdir']):
os.makedirs(f['destdir'])
f['uneval'] = "{destdir}/{nbname}.ipynb".format(**f)
f['eval'] = "{destdir}/{nbname}.eval.ipynb".format(**f)
f['py'] = "{destdir}/{nbname}.py".format(**f)
# 1. Uneval notebook
shutil.copyfile(f['nbpath'], f['uneval'])
with open(f['nbpath']) as nb_f:
nb = nbformat.read(nb_f, as_version=4)
# 2. Python
export_python(nb, f['py'])
# 3. HTML (execute first)
executer = ExecutePreprocessor(timeout=240)
executer.preprocess(nb, {})
html = export_html(nb, f)
# 4. Eval'd notebook
with open(f['eval'], 'w') as eval_f:
nbformat.write(nb, eval_f)
# Create link to notebook and script files
link_rst = "({uneval}; {eval}; {py})".format(
uneval=formatted_link(f['uneval']),
eval=formatted_link(f['eval']),
py=formatted_link(f['py']),
)
rst_file = self.state_machine.document.attributes['source']
self.state_machine.insert_input([link_rst], rst_file)
# create notebook node
nb_node = notebook_node('', html, format='html', source='nb_path')
nb_node.source, nb_node.line = (self.state_machine
.get_source_and_line(self.lineno))
# add dependency
self.state.document.settings.record_dependencies.add(f['nbpath'])
return [nb_node]
def execute_notebook(notebook_filename, notebook_filename_out, params_dict, run_path="", timeout=6000000):
notebook_fp = os.path.join(run_path, notebook_filename)
nb = read_in_notebook(notebook_fp)
new_nb = set_parameters(nb, params_dict)
ep = ExecutePreprocessor(timeout=timeout, kernel_name='python3')
try:
ep.preprocess(new_nb, {'metadata': {'path': run_path}})
except:
msg = 'Error executing the notebook "{0}".\n\n'.format(notebook_filename)
msg = '{0}See notebook "{1}" for the traceback.'.format(msg, notebook_filename_out)
print(msg)
raise
finally:
with open(notebook_filename_out, mode='wt') as f:
nbformat.write(new_nb, f)
def run_notebook(notebook_name, nb_suffix='-out', out_path='.', timeout=3600,
**execute_kwargs):
"""Runs a notebook and saves the output in the same notebook.
Arguments:
notebook_name (string): name of the notebook to be executed.
nb_suffix (string): suffix to append to the file name of the executed
notebook.
timeout (int): timeout in seconds after which the execution is aborted.
execute_kwargs (dict): additional arguments passed to
`ExecutePreprocessor`.
out_path (string): folder where to save the output notebook.
"""
timestamp_cell = "**Executed:** %s\n\n**Duration:** %d seconds."
if str(notebook_name).endswith('.ipynb'):
notebook_name = str(notebook_name)[:-len('.ipynb')]
nb_name_input = notebook_name + '.ipynb'
nb_name_output = notebook_name + '%s.ipynb' % nb_suffix
nb_name_output = os.path.join(out_path, nb_name_output)
print('- Executing: ', nb_name_input)
execute_kwargs_ = dict(kernel_name = 'python%d' % sys.version_info[0])
if execute_kwargs is not None:
execute_kwargs_.update(execute_kwargs)
ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs_)
nb = nbformat.read(nb_name_input, as_version=4)
start_time = time.time()
try:
# Execute the notebook
ep.preprocess(nb, {'metadata': {'path': './'}})
except:
# Execution failed, print a message then raise.
msg = 'Error executing the notebook "%s".\n\n' % notebook_name
msg += 'See notebook "%s" for the traceback.' % nb_name_output
print(msg)
raise
else:
# On successful execution, add timestamping cell
duration = time.time() - start_time
timestamp_cell = timestamp_cell % (time.ctime(start_time), duration)
nb['cells'].insert(0, nbformat.v4.new_markdown_cell(timestamp_cell))
finally:
# Save the notebook even when it raises an error
nbformat.write(nb, nb_name_output)
print('* Output: ', nb_name_output)
def test_tutorial_nb(file_path, workingdir, kernel=None):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
path of tutorial .ipynb file
workingdir: str
path of the directory to run the tutorial in
kernel: str
Default None
name of the kernel to use, if none, will use first kernel
in the list
"""
tutorial_name = os.path.basename(file_path)
sys.stdout.write('Testing {}...'.format(file_path))
sys.stdout.flush()
tick = time.time()
notebook = nbformat.read(file_path + '.ipynb', as_version=4)
if kernel:
eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel)
else:
eprocessor = ExecutePreprocessor(timeout=TIME_OUT)
success = True
try:
eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}})
except Exception as err:
err_msg = str(err)
fail_dict[tutorial_name] = err_msg
success = False
finally:
output_file = os.path.join(workingdir, "output.txt")
output_nb = open(output_file, mode='w')
nbformat.write(notebook, output_nb)
output_nb.close()
output_nb = open(output_file, mode='r')
for line in output_nb:
if "Warning:" in line:
success = False
if tutorial_name in fail_dict:
fail_dict[tutorial_name] += "\n"+line
else:
fail_dict[tutorial_name] = "Warning:\n"+line
sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick ))
sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed'))
sys.stdout.flush()
def nbexec(nb_base, nb):
base_name = nb_base.split('_')[1].split('.')[0].upper()
afp_name = nb.split('_')[1].split('.')[0].upper()
# Lee nb_base origen
with io.open(nb_base, 'rt', encoding="utf8") as f:
aux = nbformat.read(f, as_version=4)
# Reemplaza afp_name
u = aux['cells'][2]['source'].replace(base_name, afp_name)
aux['cells'][2]['source'] = u
# Opciones
ep = ExecutePreprocessor(timeout=300)
# Procesa aux (Revisar si preprocess modifica aux)
ep.preprocess(aux, {})
# Escribe nb destino
with io.open(nb, 'wt', encoding="utf8") as f:
nbformat.write(aux, f)
print "[INFO]--" + datetime.now().strftime('%Y-%M-%d %H:%M:%S') + "--" + "nbexec" + "--" + nb
def execute(self, write=True):
"""
Execute the specified notebook file, and optionally write out the
executed notebook to a new file.
Parameters
----------
write : bool, optional
Write the executed notebook to a new file, or not.
Returns
-------
executed_nb_path : str, ``None``
The path to the executed notebook path, or ``None`` if
``write=False``.
"""
if path.exists(self._executed_nb_path) and not self.overwrite:
logger.debug("Executed notebook already exists at {0}. Use "
"overwrite=True or --overwrite (at cmd line) to re-run"
.format(self._executed_nb_path))
return self._executed_nb_path
# Execute the notebook
logger.debug('Executing notebook using kwargs '
'"{}"...'.format(self._execute_kwargs))
executor = ExecutePreprocessor(**self._execute_kwargs)
with open(self.nb_path) as f:
nb = nbformat.read(f, as_version=IPYTHON_VERSION)
try:
executor.preprocess(nb, {'metadata': {'path': self.path_only}})
except CellExecutionError:
# TODO: should we fail fast and raise, or record all errors?
raise
if write:
logger.debug('Writing executed notebook to file {0}...'
.format(self._executed_nb_path))
with open(self._executed_nb_path, 'w') as f:
nbformat.write(nb, f)
return self._executed_nb_path
def test_tutorial_nb(file_path):
"""Run tutorial jupyter notebook to catch any execution error.
Parameters
----------
file_path : str
path of tutorial markdown file
"""
tutorial_name = os.path.basename(file_path)
notebook = nbformat.read(file_path + '_python.ipynb', as_version=4)
eprocessor = ExecutePreprocessor(timeout=900)
try:
eprocessor.preprocess(notebook, {'metadata': {}})
except Exception as err:
err_msg = "Python script successfully run without error or warning " \
"but notebook returned error:\n%s\nSomething weird happened." \
% (str(err))
fail_dict[tutorial_name] = err_msg
def run_notebook(filename):
run_path = os.path.split(filename)[0]
with open(filename) as f:
nb = nbformat.read(f, as_version=4)
try:
ep = ExecutePreprocessor(timeout=600, kernel_name='python3')
ep.preprocess(nb, {'metadata': {'path': run_path}})
# FIXME: use tempfile and mv to avoid interruption
# better split the source code of the notebook and the compiled targets.
with open(filename, 'wt') as f:
nbformat.write(nb, f)
except Exception as e:
print('processing', filename, e)
def run_notebook(self, nb, f):
"""Runs a loaded notebook file."""
if PYTHON_MAJOR_VERSION == 3:
kernel_name = 'python3'
elif PYTHON_MAJOR_VERSION == 2:
kernel_name = 'python2'
else:
raise Exception('Only Python 2 and 3 are supported')
ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name)
try:
ep.preprocess(nb, {'metadata': {'path': '.'}})
except CellExecutionError:
msg = 'Error executing the notebook "%s".\n\n' % f.name
msg += 'See notebook "%s" for the traceback.' % f.name
print(msg)
raise
finally:
nbformat.write(nb, f)
def test_notebook(notebook):
fullfile = os.path.abspath(notebook)
_, filename = os.path.split(fullfile)
filename, _ = os.path.splitext(filename)
if filename in KNOWN_FAILURES:
pytest.skip('{0} is known to fail'.format(filename))
if filename in RPY2_NOTEBOOKS and not HAS_RPY2:
pytest.skip('{0} since rpy2 is not installed'.format(filename))
if filename in JOBLIB_NOTEBOOKS and not JOBLIB_NOTEBOOKS:
pytest.skip('{0} since joblib is not installed'.format(filename))
with io.open(fullfile, encoding='utf-8') as fp:
nb = nbformat.read(fp, as_version=4)
ep = ExecutePreprocessor(allow_errors=False,
timeout=20,
kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def execute_ipynb(file_paths, exit_on_first, verbose):
failed = False
for test in file_paths:
path = pathlib.Path(test)
print()
print("=" * 40)
print(test)
with open(test) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600,
kernel_name="python3",
store_widget_state=True)
try:
ep.preprocess(nb, {"metadata": {"path": path.parent}})
except CellExecutionError:
failed = True
with atomic_write(test, mode="w") as f:
nbformat.write(nb, f)
if failed and exit_on_first:
raise SystemExit(
f"notebook execution failed in {test}, error saved "
"in notebook")
def notebook_execute(notebook_filename,
kernel_name='python3',
timeout=600,
execution_dir='./',
output_filemark="",
output_timestamp=True):
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
if output_timestamp:
output_filemark += time.strftime(".%Y.%m.%d-%H.%M")
executed_notebook_filename = "{1}{0}{2}".format(
output_filemark, *os.path.splitext(notebook_filename))
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name)
ep.preprocess(nb, {'metadata': {'path': execution_dir}})
with open(executed_notebook_filename, 'wt') as f:
nbformat.write(nb, f)
def test_invalid_notebooks(
invalid_notebook_path, cell_location, error_name, error_value, error_output_type
):
notebook_filename = script_relative_path(invalid_notebook_path)
with open(notebook_filename) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name="python3")
try:
ep.preprocess(
nb,
{
"metadata": {
"path": script_relative_path(
notebook_filename[: notebook_filename.rfind("/")]
)
}
},
)
except CellExecutionError:
error_message = get_dict_value(nb, cell_location)
assert error_message.ename == error_name
assert bool(re.search(error_value, error_message.evalue))
assert error_message.output_type == error_output_type
def test_notebooks(self, n):
nb_path = pathlib.PurePath(n)
nb_rel_path = nb_path.relative_to(BOOK_DIR)
os.chdir(str(nb_path.parent))
with open(n) as f:
try:
nb = nbformat.read(f, as_version=4)
except json.read.NotJSONError:
self.fail(f'Notebook is not valid JSON: {nb_rel_path}.\n')
except json.decoder.JSONDecodeError:
self.fail(f'Unable to parse notebook {nb_rel_path}.\n')
ep = ExecutePreprocessor(timeout=int(TIMEOUT), kernel_name=KERNEL)
try:
ep.preprocess(nb, {'metadata': {'path': RUN_PATH}})
except CellExecutionError as e:
self.fail(f'{nb_rel_path} failed due to exception.\n{e.traceback}')
except TimeoutError:
self.fail(f'Timeout executing the notebook {n}.\n')
finally:
# This is where we could optionally write the notebook to an output file
# with open(n_out + '.ipynb', mode='wt') as f:
# nbformat.write(nb, f)
pass
def test_that_examples_run(self):
cwd = os.getcwd()
os.chdir(examplesdir)
example_files = [
f for f in listdir(examplesdir) if isfile(join(examplesdir, f))
and f.endswith(".ipynb") and not f.startswith("test")
]
flag = 0
failed_examples = []
for f in example_files:
nb_name, file_extension = os.path.splitext(os.path.basename(f))
with open(f) as file:
nb = nbformat.read(file, as_version=4)
nb.metadata.get('kernelspec', {})['name'] = kernel_name
proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name)
proc.allow_errors = True
proc.preprocess(nb)
errors = []
for cell in nb.cells:
if 'outputs' in cell:
for output in cell['outputs']:
if output.output_type == 'error':
errors.append(output)
print(f, len(errors))
if errors:
failed_examples.append(f)
flag = 1
os.chdir(cwd)
if len(failed_examples) > 0:
print("failed examples: {0}".format(failed_examples))
self.assertEqual(flag, 0)
def test_notebook(path):
import nbformat
from nbconvert.preprocessors import ExecutePreprocessor
if path in SKIP:
pytest.skip("Notebook marked as 'skip'")
if path in MARS:
if not os.path.exists(os.path.expanduser("~/.ecmwfapirc")):
pytest.skip("No ~/.ecmwfapirc")
if path in CDS:
if not os.path.exists(os.path.expanduser("~/.cdsapirc")):
pytest.skip("No ~/.cdsapirc")
# if path in TENSORFLOW:
# if sys.version_info >= (3, 9):
# pytest.skip("Tensorflow not yet ready on 3.9")
with open(os.path.join(EXAMPLES, path)) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=60 * 60 * 5, kernel_name="python3")
proc.preprocess(nb, {"metadata": {"path": EXAMPLES}})
def test_sd_roi(hdf5_ds_2, tmpdir_factory, lt_ctx, local_cluster_url):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
roi_params = {"shape": "rect", "x": 1, "y": 2, "width": 6, "height": 6}
analysis = [{
"analysisType": 'SD_FRAMES',
"parameters": {
"roi": roi_params
}
}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
ep.preprocess(nb, {"metadata": {"path": datadir}})
data_path = os.path.join(datadir, 'sd_result.npy')
results = np.load(data_path)
nx, ny = hdf5_ds_2.shape.nav
roi = masks.rectangular(X=roi_params["x"],
Y=roi_params["y"],
Width=roi_params["width"],
Height=roi_params["height"],
imageSizeX=nx,
imageSizeY=ny)
udf = StdDevUDF()
expected = lt_ctx.run_udf(dataset=hdf5_ds_2, udf=udf, roi=roi)
assert np.allclose(
results,
expected['varsum'].raw_data,
)
def run_notebook(notebook_path):
# From https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/
nb_name, _ = os.path.splitext(os.path.basename(notebook_path))
dirname = os.path.dirname(notebook_path)
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=600, kernel_name="python3")
proc.allow_errors = True
proc.preprocess(nb, {})
output_path = os.path.join(dirname, "{}_all_output.ipynb".format(nb_name))
with open(output_path, mode="wt") as f:
nbformat.write(nb, f)
errors = []
for cell in nb.cells:
if "outputs" in cell:
for output in cell["outputs"]:
if output.output_type == "error":
errors.append(output)
os.remove(output_path)
return nb, errors
def run_notebook(notebook_path):
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=600, kernel_name="python3")
proc.allow_errors = True
script_path = os.path.dirname(os.path.abspath(__file__))
package_path = os.path.abspath(os.path.join(script_path, "..", ".."))
# Add shutdown for show method.
shutdown_cell = new_code_cell(
"from interpret import shutdown_show_server\nshutdown_show_server()")
nb.cells.append(shutdown_cell)
proc.preprocess(nb, {"metadata": {"path": package_path}})
errors = []
for cell in nb.cells:
if "outputs" in cell:
for output in cell["outputs"]:
if output.output_type == "error":
errors.append(output)
return nb, errors
def run_notebook(notebook_path, timeout=hours(6), save_nb_path=None):
nb_name, _ = os.path.splitext(os.path.basename(notebook_path))
dirname = os.path.dirname(notebook_path)
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=timeout, kernel_name='python3')
proc.allow_errors = True
proc.preprocess(nb, {'metadata': {'path': dirname}})
if save_nb_path:
with open(save_nb_path, mode='wt') as f:
nbformat.write(nb, f)
errors = []
for cell in nb.cells:
if 'outputs' in cell:
for output in cell['outputs']:
if output.output_type == 'error':
errors.append(output)
return nb, errors
def run_notebook(notebook_path):
nb_name, _ = os.path.splitext(os.path.basename(notebook_path))
dirname = os.path.dirname(notebook_path)
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
proc = ExecutePreprocessor(timeout=600, kernel_name='python3')
proc.allow_errors = True
proc.preprocess(nb, {'metadata': {'path': './'}})
#output_path = os.path.join(dirname, '{}_all_output.ipynb'.format(nb_name))
#with open(output_path, mode='wt') as f:
# nbformat.write(nb, f)
errors = []
for cell in nb.cells:
if 'outputs' in cell:
for output in cell['outputs']:
if output.output_type == 'error':
errors.append(output)
return nb, errors
def runtest(self):
self._skip()
with io.open(self.name,encoding='utf8') as nb:
notebook = nbformat.read(nb, as_version=4)
# TODO: which kernel? run in pytest's or use new one (make it option)
_timeout = self.parent.parent.config.getini('nbsmoke_cell_timeout')
kwargs = dict(timeout=int(_timeout) if _timeout!='' else 300,
allow_errors=False,
# or sys.version_info[1] ?
kernel_name='python')
ep = ExecutePreprocessor(**kwargs)
with cwd(os.path.dirname(self.name)): # jupyter notebook always does this, right?
ep.preprocess(notebook,{})
# TODO: clean up this option handling
if self.parent.parent.config.option.store_html != '':
he = nbconvert.HTMLExporter()
# could maybe use this for chance of testing the html? but not the aim of this project
#he.template_file = 'basic'
html, resources = he.from_notebook_node(notebook)
with io.open(os.path.join(self.parent.parent.config.option.store_html,os.path.basename(self.name)+'.html'),'w',encoding='utf8') as f:
f.write(html)
def execute(path_nb,
dated=True,
timeout=600,
kernel_name='python3',
verbose=False):
"""
"""
folder = os.path.dirname(path_nb)
filename = os.path.basename(path_nb)
now = dt.datetime.now().strftime('%Y%m%d_%H%M%S.%f')[:-3]
if dated:
save_name = '{}_{}'.format(now, filename)
else:
save_name = filename
# path_output
path_out = os.path.join(folder, save_name)
# load nb
with open(path_nb, 'r', encoding='utf-8') as f:
nb = nbformat.read(f, as_version=4)
# create ep
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name)
# run catching errors
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
out = ep.preprocess(nb, {'metadata': {'path': '.'}})
except CellExecutionError:
out = None
msg = 'Error executing notebook {}\nSee notebook {} for traceback.'
msg = msg.format(path_nb, path_out)
print(msg)
raise
finally:
with open(path_out, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
if verbose:
md = 'Notebook **{}** run and saved as **{}**'
md = md.format(path_nb, path_out)
display(Markdown(md))
return path_out, now
def run_notebook(
*,
notebook_name=None,
notebook_path=None,
output_notebook_name=None,
output_notebook_path=None,
timeout=-1,
notebook_version=4,
kernel='python3',
):
"""Execute a jupyter notebook
kernel name is an issue: https://github.com/jupyter/nbconvert/issues/515
"""
if notebook_path is None:
notebook_path = paths['notebook_path']
else:
notebook_path = pathlib.Path(notebook_path)
if output_notebook_path is None:
output_notebook_path = paths['interim_data_path']
else:
output_notebook_path = pathlib.Path(output_notebook_path)
if output_notebook_name is None:
output_notebook_name = notebook_name
output_notebook_fq = output_notebook_path / output_notebook_name
with open(notebook_path / notebook_name) as f:
nb = nbformat.read(f, as_version=notebook_version)
ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
try:
out = ep.preprocess(nb, {'metadata': {'path': notebook_path}})
except CellExecutionError:
out = None
msg = f"""Error executing the notebook "{notebook_name}".
See notebook "{str(output_notebook_fq)}" for the traceback.'
"""
logger.error(msg)
raise
finally:
with open(output_notebook_fq, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
return output_notebook_name
def run_notebook(nb_wd, nb_path):
"""Runs a given notebook and saves it
Executes the passed notebook with nb_wd as the working directory.
If an error occurs during the execution, a message is raised to the user
and the notebook is saved anyway, including the traceback.
Parameters
----------
nb_wd : Path or str
Path to the folder which should be used as a working directory for
the execution of the notebook
nb_path : Path or str
Full path to the notebook which should be run.
Returns
-------
Nothing
"""
if not isinstance(nb_wd, Path):
nb_wd = Path(nb_wd)
if not isinstance(nb_path, Path):
nb_path = Path(nb_path)
with nb_path.open() as f:
nb = nbformat.read(f, as_version=nbformat.NO_CONVERT)
# Configure notebook execution mode
# Timeout = None means no restriction on runtime of cells
ep = ExecutePreprocessor(timeout=None, kernel_name='python3')
# The code for the following error handling is taken from the
# official nbconvert documentation:
# https://nbconvert.readthedocs.io/en/latest/execute_api.html
try:
# Run notebook
out = ep.preprocess(nb, {'metadata': {'path': str(nb_wd)}})
except CellExecutionError:
out = None
msg = f'Error executing the notebook "{str(nb_path)}".\n\n'
msg += 'See the notebook for the traceback.\n'
print(msg)
raise
finally:
# Save it. Includes tracebacks should an error have occured.
with nb_path.open('wt') as f:
nbformat.write(nb, f)
return
def execute_notebook(notebook_file_path: str, output_file_folder: str):
file_name = os.path.basename(os.path.normpath(notebook_file_path))
# Read notebook
with open(notebook_file_path) as f:
nb = nbformat.read(f, as_version=4)
has_error = False
# Execute notebook
try:
# Create preprocessors
remove_no_execute_cells_preprocessor = RemoveNoExecuteCells()
execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3")
# Use no-execute preprocessor
(
nb_no_execute_cells_removed,
resources_no_execute_cells_removed,
) = remove_no_execute_cells_preprocessor.preprocess(nb)
# Execute notebook
out = execute_preprocessor.preprocess(
nb_no_execute_cells_removed, resources_no_execute_cells_removed
)
except Exception as error:
out = None
print(f"Error executing the notebook: {notebook_file_path}.\n\n")
has_error = True
raise
finally:
output_file_path = os.path.join(
output_file_folder, "failure" if has_error else "success", file_name
)
# Create directories if they don't exist
if not os.path.exists(os.path.dirname(output_file_path)):
try:
os.makedirs(os.path.dirname(output_file_path))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
print(f"Writing output to: {output_file_path}")
with open(output_file_path, mode="w", encoding="utf-8") as f:
nbformat.write(nb, f)
def test_func(self):
passing = True
print("\n--------------- Testing {0} ---------------".format(nbname))
print(" {0}".format(nbpath))
if nbname in py2Ignore and sys.version_info[0] == 2:
print(" Skipping {}".format(nbname))
return
ep = ClearOutputPreprocessor()
with open(nbpath) as f:
nb = nbformat.read(f, as_version=4)
ep.preprocess(nb, {})
ex = ExecutePreprocessor(timeout=600,
kernel_name='python{}'.format(
sys.version_info[0]),
allow_errors=True)
out = ex.preprocess(nb, {})
for cell in out[0]['cells']:
if 'outputs' in cell.keys():
for output in cell['outputs']:
if output[
'output_type'] == 'error': #in output.keys():
passing = False
err_msg = []
for o in output['traceback']:
err_msg += ["{}".format(o)]
err_msg = "\n".join(err_msg)
msg = """
\n <<<<< {} FAILED >>>>> \n
{} in cell [{}] \n-----------\n{}\n-----------\n
----------------- >> begin Traceback << ----------------- \n
{}\n
\n----------------- >> end Traceback << -----------------\n
""".format(nbname, output['ename'],
cell['execution_count'], cell['source'],
err_msg)
assert passing, msg
print("\n ..... {0} Passed ..... \n".format(nbname))
def nb_task(ds, **kwargs):
notebook_dir = os.path.dirname(notebook_path)
with open(notebook_path) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=21600)
try:
out = ep.preprocess(nb, {'metadata': {'path': notebook_dir}})
except CellExecutionError:
msg = 'Error executing the notebook "%s".\n\n' % notebook_path
msg += 'See notebook "%s" for the traceback.' % notebook_path
print(msg)
raise
finally:
with open(notebook_path, mode='wt') as f:
nbformat.write(nb, f)
def exec_notebook(self, nb, resources, nb_dir):
nb_copy, _ = AddEpilog(config=self.config, pkg_dir=self.output_dir).preprocess(nb, resources)
resources['outputs']['notebooks/executed-source.ipynb'] = nbformat.writes(nb).encode('utf-8')
exec_nb_path = self.write_notebook(nb_copy)
try:
ep = ExecutePreprocessor(config=self.config)
nb, _ = ep.preprocess(nb, {'metadata': {'path': nb_dir}})
except CellExecutionError as e:
raise CellExecutionError("Errors executing noteboook. See output at {} for details.\n{}"
.format(exec_nb_path, ''))
return nb, resources
def test_monitor_cell(self):
expected_logs = [
"Beginning machine learning...",
"Still training...",
"Done!",
# Handle unicode
"хнЧ"
]
with open("./tests/jupyter_test_file.py.ipynb") as f:
nb = nbformat.read(f, as_version=4.0)
ep = ExecutePreprocessor(timeout=5000, kernel_name="python")
result = ep.preprocess(nb, {})
# Accumulate all output from cell 2
all_stdout = ""
second_cell_output = result[0]["cells"][1]["outputs"]
for output in second_cell_output:
if output.name == "stdout":
all_stdout = all_stdout + output.text
# Assert on all output from cell2
for log in expected_logs:
if PY2:
assert_in(log, all_stdout.encode("utf-8"))
continue
assert_in(log, all_stdout)
# Verify that variables declared in previous cells can be affected
third_cell_output = result[0]["cells"][2]["outputs"]
assert third_cell_output[0].text == "a=1\n"
# Make sure logs were persisted
log_dir = get_hyperdash_logs_home_path_for_job("test_jupyter")
latest_log_file = max([
os.path.join(log_dir, filename) for
filename in
os.listdir(log_dir)
], key=os.path.getmtime)
with open(latest_log_file, 'r') as log_file:
data = log_file.read()
for log in expected_logs:
assert_in(log, data)
os.remove(latest_log_file)
def grade_notebook(path, notebook_path, output_path, kernal='python3'):
print("Running notebooks: ", notebook_path)
with open(notebook_path) as f:
nb = nbf.read(f, as_version=4)
ep = ExecutePreprocessor(allow_errors=True,timeout=600, kernel_name=kernal)
try:
out = ep.preprocess(nb, {'metadata': {'path': path}})
except CellExecutionError:
out = None
msg = 'Error executing the notebook "%s".\n\n' % notebook_filename
msg += 'See notebook "%s" for the traceback.' % output_path
print(msg)
raise
finally:
print("Writing to", output_path)
with open( output_path, mode='w', encoding='utf-8') as f:
nbf.write(nb, f)
return 4, "4. Graded."
def retrieve_errors(nb_path):
"""
Find all errors that occur when running a nb
"""
with nb_path.open() as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name='python3',
allow_errors=True)
out, _ = ep.preprocess(nb, {'metadata': {'path': '.'}})
errors = []
for cell in out.cells:
if "outputs" in cell:
for output in cell["outputs"]:
if output.output_type == "error":
errors.append(output.evalue)
return errors
def do_execute(self, script, kernel, timeout=600):
cells = [nbformat.v4.new_code_cell(source=script)]
nb = nbformat.v4.new_notebook(cells=cells,
metadata={'language': 'python'})
# Execute the script
executor = ExecutePreprocessor(timeout=timeout, kernel_name=kernel)
nb, metadata = executor.preprocess(nb, {'metadata': {}})
# Export html from the notebook
html_exporter = HTMLExporter()
html_exporter.template_file = 'basic'
body, _ = html_exporter.from_notebook_node(nb)
# Extract the result part of the html
output = ScriptExecutor.extract_output(body, kernel)
return output
def test_radial_fourier_default(hdf5_ds_2, tmpdir_factory, lt_ctx):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'local'}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{
"analysisType": "RADIAL_FOURIER",
"parameters": {
'shape': 'radial_fourier',
'cx': 0,
'cy': 0,
'ri': 0,
'ro': 2,
'n_bins': 2,
'max_order': 23,
}
}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
out = ep.preprocess(nb, {"metadata": {"path": datadir}})
channels = ["dominant_0", "absolute_0_0", "absolute_0_1"]
results = {}
for channel in channels:
data_path = os.path.join(datadir, f"radial_result_{channel}.npy")
results[channel] = np.load(data_path)
analysis = lt_ctx.create_radial_fourier_analysis(dataset=hdf5_ds_2,
cx=0,
cy=0,
ri=0,
ro=2,
n_bins=2,
max_order=23)
expected = lt_ctx.run(analysis)
assert np.allclose(results["dominant_0"], expected["dominant_0"].raw_data)
assert np.allclose(results["absolute_0_0"],
expected["absolute_0_0"].raw_data)
assert np.allclose(results["absolute_0_1"],
expected["absolute_0_1"].raw_data)
def test_sum_default(hdf5_ds_2, tmpdir_factory):
datadir = tmpdir_factory.mktemp('template_tests')
conn = {'connection': {'type': 'local'}}
path = hdf5_ds_2.path
dataset = _get_hdf5_params(path)
analysis = [{"analysisType": "SUM_FRAMES", "parameters": {"roi": {}}}]
notebook = notebook_generator(conn, dataset, analysis, save=True)
notebook = io.StringIO(notebook.getvalue())
nb = nbformat.read(notebook, as_version=4)
ep = ExecutePreprocessor(timeout=600)
out = ep.preprocess(nb, {"metadata": {"path": datadir}})
data_path = os.path.join(datadir, 'sum_result.npy')
result = np.load(data_path)
with hdf5_ds_2.get_reader().get_h5ds() as h5ds:
data = h5ds[:]
expected = data.sum(axis=(0, 1))
assert np.allclose(expected, result)
def generate_html(self, kernel_name=""):
nb_executor = ExecutePreprocessor(kernel_name=kernel_name)
nb_executor.allow_errors = True
html_exporter = HTMLExporter()
html_writer = FilesWriter()
with open(self.output_ipynb_path) as f:
nb = nbformat.read(f, as_version=4)
nb = nb_executor.preprocess(nb)[0]
body, resources = html_exporter.from_notebook_node(nb)
# HTML writer automatically adds .html to the end, so get rid of it
html_writer.write(
body, resources, self.output_html_path.replace(".html", "")
)
print(f"Written file to {self.output_html_path}") if self.verbose else None
self.metadata["html-file"] = os.path.basename(self.output_html_path)
self.metadata.mark_as_updated()
def execute_notebook(src='estimate.src.ipynb', dest='estimate.ipynb'):
"""Executes the analysis notebook and writes out a copy with all of the
resulting tables and plots.
Parameters
----------
src: str, optional
Source notebook to execute
dest: str, optional
Output notebook
"""
with open(src) as fp:
nb = nbformat.read(fp, 4)
exp = ExecutePreprocessor(timeout=300)
updated_nb, _ = exp.preprocess(nb, {})
with open(dest, 'w') as fp:
nbformat.write(updated_nb, fp)
def execute_notebook(input_file: Path,
output_file: Path) -> Dict:
''' Execute notebook within a folder
Parameters
----------
input_file
notebook file to be executed
output_file
notebook execution output file
Returns
-------
a pandas dataframe
contains filename and error(s) encountered.
'''
error = {}
with open(input_file) as f:
try:
nb = nbformat.read(f, as_version=4)
kernel = nb.dict()['metadata']['kernelspec']['name']
logger.info(f'Running... {input_file.stem}')
ep = ExecutePreprocessor(timeout=600, kernel_name=kernel)
out = ep.preprocess(nb, {'metadata': {'path': input_file.parent}})
except CellExecutionError:
out = None
msg = f'Error executing notebook".\n'
msg += f'See notebook "{output_file}" for the traceback.'
logger.info(msg)
error = {'error': msg, 'file': input_file}
finally:
with open(output_file, mode='w', encoding='utf-8') as f:
nbformat.write(nb, f)
return error
def execute(nb_in_fn, kernel_name='python3', run_path='.'):
nb_out_fn = nb_in_fn.replace('.ipynb', '.nbconvert.ipynb')
with open(nb_in_fn) as f:
nb = nbformat.read(f, as_version=4)
ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name)
failed = False
try:
out = ep.preprocess(nb, {'metadata': {'path': run_path}})
except CellExecutionError as e:
out = None
print('Error executing the notebook "{}". Traceback:'.format(nb_in_fn))
print(e.traceback)
failed = True
finally:
with open(nb_out_fn, mode='wt') as f:
nbformat.write(nb, f)
return not failed
def run_notebook_with_args(input_notebook, output_notebook, unknown):
with codecs.open(input_notebook, encoding="utf-8") as f:
nb = nbformat.read(f, as_version=4)
orig_parameters = extract_parameters(nb)
params = []
args = compute_args(unknown)
for k, v in args.items():
for p in orig_parameters:
if p.name == k:
params.append(p.with_value(v))
if len(params) > 0:
nb = replace_definitions(nb, params, execute=False)
ep = ExecutePreprocessor(timeout=None, kernel_name='python3')
out = ep.preprocess(nb, {'metadata': {'path': '.'}})
with codecs.open(output_notebook, encoding="utf-8", mode="w") as f:
nbformat.write(nb, f)
