def nb_run(fname):
""" given a valid notebook path, run and save notebook """
# initialize error flag
err = False
# file name
fn = fname
# run notebook
try:
notebook = read(open(fn), 'json')
r = NotebookRunner(notebook)
r.run_notebook()
err = False
except Exception as e:
logger.error(e)
logger.debug('failed to run notebook')
err = True
# save notebook
try:
write(r.nb, open(fn, 'w'), 'json')
err = False
except Exception as e:
logger.error(e)
logger.debug('failed to save notebook')
err = True
return err
def run_notebooks(selected_nb_re=None):
""" Run the tutorial notebooks. """
from runipy.notebook_runner import NotebookRunner
_orig_path = os.getcwd()
# walk through each directory in tutorials/ to find all .ipynb file
for tutorial_filename,nb in walk_through_tutorials(only_published=True,
selected_nb_re=selected_nb_re):
path,filename = os.path.split(tutorial_filename)
if filename.startswith("_run_"):
continue
logger.info("Running tutorial: {}".format(filename))
# notebook file
output_filename = os.path.join(path,"_run_{}"
.format(filename))
# prepend _run_ to the notebook names to create new files
# so the user isn't left with a bunch of modified files.
os.chdir(path)
r = NotebookRunner(nb, mpl_inline=True)
r.run_notebook(skip_exceptions=True)
write(r.nb, open(output_filename, 'w'), 'json')
os.chdir(_orig_path)
def test_nbs():
path = os.path.join(pyfolio_root(), "examples", "*.ipynb")
for ipynb in glob.glob(path):
with open(ipynb) as f:
nb = read_notebook(f, "json")
nb_runner = NotebookRunner(nb)
nb_runner.run_notebook(skip_exceptions=False)
def test_notebook(filename):
nb = read(open(filename), 'json')
expected = deepcopy(nb['worksheets'][0]['cells'])
# TODO: figure out how to set the log-level for IPython here
r = NotebookRunner(nb)
r.run_notebook()
actual = r.nb['worksheets'][0]['cells']
failed = []
for exp, act in zip(expected, actual):
if exp['cell_type'] == 'code':
#print "comparing outputs for ", exp['input']
for eo,ao in zip(exp['outputs'], act['outputs']):
#print "\t", eo['text'], ao['text'], eo['text']==ao['text']
eotext = ['\n'.join(l for l in eo['text'].split('\n') if _filter_exceptions(l))]
aotext = ['\n'.join(l for l in ao['text'].split('\n') if _filter_exceptions(l))]
if eotext != aotext:
#print "\tFAILED"
failed.append({'prompt_number': exp['prompt_number'], 'input': exp['input'], 'expected_output': eotext, 'actual_output': aotext})
return len(failed)==0, failed, r.nb
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=True):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
curr_dir = os.getcwd()
os.chdir(os.path.dirname(nb_path))
notebook = read(open(nb_path), 'json')
nb_runner = NotebookRunner(notebook, pylab=True)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print ''
print e
# Return the traceback, filtering out ANSI color codes.
# http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences
return 'Notebook conversion failed with the following traceback: \n%s' % \
re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e))
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
os.chdir(curr_dir)
return ret
def testRunNotebooks(self):
notebook_dir = path.join('tests', 'input')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print(notebook_file)
expected_file = path.join('tests', 'expected', notebook_file)
notebook = ""
with open(notebook_path) as notebook_file:
notebook = notebook_file.read()
try:
# IPython 3
notebook = reads(notebook, 3)
except (TypeError, NBFormatError):
# IPython 2
notebook = reads(notebook, 'json')
runner = NotebookRunner(notebook, working_dir=notebook_dir)
runner.run_notebook(True)
expected = ""
with open(expected_file) as notebook_file:
expected = notebook_file.read()
try:
# IPython 3
expected = reads(expected, 3)
except (TypeError, NBFormatError):
# IPython 2
expected = reads(expected, 'json')
self.assert_notebooks_equal(expected, runner.nb)
def test_nbs():
path = os.path.join(pyfolio_root(), 'examples', '*.ipynb')
for ipynb in glob.glob(path):
with open(ipynb) as f:
nb = read(f, 'json')
nb_runner = NotebookRunner(nb)
nb_runner.run_notebook(skip_exceptions=False)
def run_ipython_notebook(notebook_str):
"""
References:
https://github.com/paulgb/runipy
>>> from utool.util_ipynb import * # NOQA
"""
from runipy.notebook_runner import NotebookRunner
import nbformat
import logging
log_format = '%(asctime)s %(levelname)s: %(message)s'
log_datefmt = '%m/%d/%Y %I:%M:%S %p'
logging.basicConfig(
level=logging.INFO, format=log_format, datefmt=log_datefmt
)
#fpath = 'tmp.ipynb'
#notebook_str = ut.readfrom(fpath)
#nb3 = IPython.nbformat.reads(notebook_str, 3)
#cell = nb4.cells[1]
#self = runner
#runner = NotebookRunner(nb3, mpl_inline=True)
print('Executing IPython notebook')
nb4 = nbformat.reads(notebook_str, 4)
runner = NotebookRunner(nb4)
runner.run_notebook(skip_exceptions=False)
run_nb = runner.nb
return run_nb
def test_tutorial_notebook():
from IPython.nbformat.current import read
from runipy.notebook_runner import NotebookRunner
rootdir = os.path.join(aospy.__path__[0], 'examples')
with open(os.path.join(rootdir, 'tutorial.ipynb')) as nb:
notebook = read(nb, 'json')
r = NotebookRunner(notebook)
r.run_notebook()
r.shutdown_kernel()
def testRunNotebooks(self):
input_glob = path.join('tests', 'input', '*.ipynb')
for notebook_file in glob(input_glob):
notebook_file_base = path.basename(notebook_file)
print notebook_file_base
expected_file = path.join('tests', 'expected', notebook_file_base)
runner = NotebookRunner(notebook_file)
runner.run_notebook(True)
expected = read(open(expected_file), 'json')
self.assert_notebooks_equal(expected, runner.nb)
def testRunNotebooks(self):
notebook_dir = path.join('tests', 'input')
for notebook_path in glob(path.join(notebook_dir, '*.ipynb')):
notebook_file = path.basename(notebook_path)
print notebook_file
expected_file = path.join('tests', 'expected', notebook_file)
runner = NotebookRunner(read(open(notebook_path), 'json'), working_dir=notebook_dir)
runner.run_notebook(True)
expected = read(open(expected_file), 'json')
self.assert_notebooks_equal(expected, runner.nb)
def evaluate_notebook(nb_path, dest_path=None):
# Create evaluated version and save it to the dest path.
nb_runner = NotebookRunner(nb_in=nb_path)
nb_runner.run_notebook()
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
nb_runner.save_notebook(dest_path)
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
def check_ipython_notebook():
notebooks = glob.glob("*ipynb")
N = len(notebooks)
pb = Progress(N)
for i, filename in enumerate(notebooks):
print(purple(filename))
notebook = read(open(filename), 'json')
r = NotebookRunner(notebook)
r.run_notebook()
pb.animate(i + 1)
def run(nbtxt, output=None, view=False):
""" Run a notebook app 100% from JSON (text stream), return the JSON (text stream)
:param nbtxt: JSON representation of notebook app, ready to run
:param view: don't invoke notebook, just view in current form
NOTE: `view` probably isn't useful, since the input will just be output again
"""
# TODO: support output parameter to specify only returning certain attributes from notebook
# create a notebook object from the JSON
nb_obj = nb_read_json(nbtxt)
nb_runner = NotebookRunner(nb_obj)
try: # get the app name from metadata
name = nb_obj['metadata']['conda.app']['name']
except KeyError as ex:
name = "nbapp"
try:
if view:
pass # then don't run it
else:
nb_runner.run_notebook(skip_exceptions=False)
return nb_runner.nb
except Empty as ex:
sys.stderr.write("IPython Kernel timeout")
err = mini_markdown_nb("""
Notebook Error
==============
ERROR: IPython Kernel timeout
```
{error}
```
""".format(error=str(ex).split(':')[-1]))
return err
except (NotImplementedError, NotebookError, ValueError) as ex:
msg = str(ex).splitlines()[-1]
sys.stderr.write(msg)
err = mini_markdown_nb("""
Notebook Error
==============
Notebook contains unsupported feature or bad argument:
```
{error}
```
""".format(error=msg))
return err
except ImportError:
msg = "nodejs or pandoc must be installed"
sys.stderr.write(msg)
err = mini_markdown_nb(msg)
return err
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
nb_runner = NotebookRunner(nb_in=nb_path, pylab=True)
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
nb_runner.save_notebook(dest_path)
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
def run_notebook(nbpath):
logging.info("Reading notebook '%s'", nbpath)
with open(nbpath) as nbfile:
notebook = nbformat.read(nbfile, as_version=3)
runner = NotebookRunner(notebook)
try:
runner.run_notebook()
except NotebookError:
logging.error("An error occurred while executing notebook '%s'. "
"Exiting with nonzero exit status", nbpath)
sys.exit(1)
def run_notebook(nbpath):
logging.info("Reading notebook '%s'", nbpath)
with open(nbpath) as nbfile:
notebook = read(nbfile, 'json')
runner = NotebookRunner(notebook, mpl_inline=True)
try:
runner.run_notebook()
except NotebookError:
logging.error(
"An error occurred while executing notebook '%s'. "
"Exiting with nonzero exit status", nbpath)
sys.exit(1)
def evaluate_notebook(nb_path, dest_path=None):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
nb_json = nbr(open(nb_path), 'json')
nb_runner = NotebookRunner(nb_json, pylab=True)
nb_runner.run_notebook()
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
#nb_runner.save_notebook(dest_path)
nbw(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
def test_nbs():
path = os.path.join(pyfolio_root(), 'examples', '*.ipynb')
for ipynb in glob.glob(path):
# See if bayesian is useable before we run a test
if ipynb.endswith('bayesian.ipynb'):
try:
import pymc3 # NOQA
except:
continue
with open(ipynb) as f:
nb = read_notebook(f, 'json')
nb_runner = NotebookRunner(nb)
nb_runner.run_notebook(skip_exceptions=False)
def collect(self):
with self.fspath.open() as f: payload = f.read()
self.nb = nbformat.reads(payload, 3)
# kernel needs to start from the same dir the ipynb is in
notebook_dir = self.fspath.dirname
cwd = os.getcwd()
if cwd != notebook_dir: os.chdir(notebook_dir)
self.runner = NotebookRunner(self.nb)
os.chdir(cwd)
cell_num = 1
for cell in self.runner.iter_code_cells():
yield IPyNbCell.from_parent(self, name=self.name, cell_num=cell_num, cell=cell)
cell_num += 1
def test(self):
payload = open(filename)
nb = read(payload, 'json')
# turn off colors, so we can properly read exceptions
cell_nc = new_code_cell(input='%colors NoColor', prompt_number=0)
nb.worksheets[0].cells.insert(0, cell_nc)
# workaround for matplotlib backend
cell_mpl = new_code_cell(input="import matplotlib; matplotlib.use('Agg')", prompt_number=1)
nb.worksheets[0].cells.insert(1, cell_mpl)
# set working dir to notebook path
wd = os.path.abspath(os.path.dirname(filename))
runner = NotebookRunner(nb, working_dir=wd)
#try:
runner.run_notebook(skip_exceptions=False)
def run(self):
# Now convert the lecture notes, problem sets, and practice problems to
# HTML notebooks.
from runipy.notebook_runner import NotebookRunner
start_dir = os.path.abspath('.')
for notebook in glob.glob('*.ipynb'):
if os.path.dirname(notebook):
os.chdir(os.path.dirname(notebook))
r = NotebookRunner("./" + os.path.basename(notebook))
r.run_notebook(skip_exceptions=True)
r.save_notebook("./" + os.path.basename(notebook))
os.chdir(start_dir)
def collect(self):
with self.fspath.open() as f:
payload = f.read()
self.notebook_folder = self.fspath.dirname
try:
# Ipython 3
self.nb = reads(payload, 3)
except (TypeError, NBFormatError):
# Ipython 2
self.nb = reads(payload, 'json')
self.runner = NotebookRunner(self.nb)
cell_num = 1
for cell in self.runner.iter_code_cells():
yield IPyNbCell(self.name, self, cell_num, cell)
cell_num += 1
def execute_notebook(nb, resources):
"""Execute notebook. With ExecutePreprocessor using IPython >= 3 or runipy instead.
"""
if is_ipython_3():
from IPython.nbconvert.preprocessors import ExecutePreprocessor
nb, resources = ExecutePreprocessor().preprocess(nb, resources)
elif runipy_available:
from runipy.notebook_runner import NotebookRunner
r = NotebookRunner(nb)
r.run_notebook(skip_exceptions=True)
nb = r.nb
else:
raise ImportError(
"Can't execute notebooks. Please install IPython >= 3 or runipy.")
return nb
def render_nb():
# need ipynb v3 to play nice with runipy
notebook = read(open("stock_infos.ipynb"), 3)
nb = NotebookRunner(notebook, pylab=True)
nb.run_notebook()
# need ipynb v4 to play nice with Jupyter
nb = nbformat.convert(nb.nb, 4)
html_exporter = HTMLExporter()
body, resources = html_exporter.from_notebook_node(nb)
html_file = open("static/stock_infos.html", "w")
html_file.write(body.encode('utf8', 'ignore'))
html_file.close()
return app.send_static_file('stock_infos.html')
def run(self):
# Now convert the lecture notes, problem sets, and practice problems to
# HTML notebooks.
from runipy.notebook_runner import NotebookRunner
from IPython.nbformat.current import read, write
start_dir = os.path.abspath('.')
for notebook in glob.glob('notebooks/*.ipynb'):
print("Running {0}...".format(notebook))
os.chdir(os.path.dirname(notebook))
with open(os.path.basename(notebook)) as f:
r = NotebookRunner(read(f, 'json'), pylab=False)
r.run_notebook(skip_exceptions=True)
with open(os.path.basename(notebook), 'w') as f:
write(r.nb, f, 'json')
os.chdir(start_dir)
def run(self):
# Now convert the lecture notes, problem sets, and practice problems to
# HTML notebooks.
from runipy.notebook_runner import NotebookRunner
start_dir = os.path.abspath('.')
for notebook in (glob.glob('lectures/*.ipynb')
+ glob.glob('problems/*.ipynb')
+ glob.glob('practice/*.ipynb')):
if "Understanding" in notebook:
continue
os.chdir(os.path.dirname(notebook))
r = NotebookRunner(os.path.basename(notebook), pylab=True)
r.run_notebook(skip_exceptions=True)
r.save_notebook(os.path.basename(notebook))
os.chdir(start_dir)
def notebooks(env, loader, args):
nb_path = env.resolve('notebooks')
res_path = env.resolve('results')
for filename in os.listdir(nb_path):
if filename.endswith('.ipynb'):
nb_name = filename.split('.')[0]
print('running', nb_name)
src_path = os.path.join(nb_path, filename)
html_path = os.path.join(res_path, nb_name + '.html')
with open(src_path, 'r') as f:
notebook = read(f, 'json')
r = NotebookRunner(notebook, working_dir=nb_path)
r.run_notebook()
dest_path = tempfile.mkstemp()
t = tempfile.NamedTemporaryFile()
with open(t.name, 'w') as g:
write(r.nb, g)
exporter = nbconvert.HTMLExporter()
body, resources = exporter.from_filename(t.name)
with open(html_path, 'wb') as g:
g.write(body.encode('utf-8'))
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
# Always use --pylab so figures appear inline
# perhaps this is questionable?
notebook = read(open(nb_path), 'json')
nb_runner = NotebookRunner(notebook, pylab=True, mpl_inline=True)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print('\n', '-' * 80)
print(e)
print('-' * 80)
raise
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False):
# Create evaluated version and save it to the dest path.
notebook = nbformat.read(open(nb_path), 'json')
nb_runner = NotebookRunner(notebook, pylab=False)
try:
nb_runner.run_notebook(skip_exceptions=skip_exceptions)
except NotebookError as e:
print('')
print(e)
# Return the traceback, filtering out ANSI color codes.
# http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences
return "Notebook conversion failed with the " \
"following traceback: \n%s" % \
re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '',
str(e))
if dest_path is None:
dest_path = 'temp_evaluated.ipynb'
nbformat.write(nb_runner.nb, open(dest_path, 'w'), 'json')
ret = nb_to_html(dest_path)
if dest_path is 'temp_evaluated.ipynb':
os.remove(dest_path)
return ret
