def nb_run(fname): """ given a valid notebook path, run and save notebook """ # initialize error flag err = False # file name fn = fname # run notebook try: notebook = read(open(fn), 'json') r = NotebookRunner(notebook) r.run_notebook() err = False except Exception as e: logger.error(e) logger.debug('failed to run notebook') err = True # save notebook try: write(r.nb, open(fn, 'w'), 'json') err = False except Exception as e: logger.error(e) logger.debug('failed to save notebook') err = True return err
def run_notebooks(selected_nb_re=None): """ Run the tutorial notebooks. """ from runipy.notebook_runner import NotebookRunner _orig_path = os.getcwd() # walk through each directory in tutorials/ to find all .ipynb file for tutorial_filename,nb in walk_through_tutorials(only_published=True, selected_nb_re=selected_nb_re): path,filename = os.path.split(tutorial_filename) if filename.startswith("_run_"): continue logger.info("Running tutorial: {}".format(filename)) # notebook file output_filename = os.path.join(path,"_run_{}" .format(filename)) # prepend _run_ to the notebook names to create new files # so the user isn't left with a bunch of modified files. os.chdir(path) r = NotebookRunner(nb, mpl_inline=True) r.run_notebook(skip_exceptions=True) write(r.nb, open(output_filename, 'w'), 'json') os.chdir(_orig_path)
def test_nbs(): path = os.path.join(pyfolio_root(), "examples", "*.ipynb") for ipynb in glob.glob(path): with open(ipynb) as f: nb = read_notebook(f, "json") nb_runner = NotebookRunner(nb) nb_runner.run_notebook(skip_exceptions=False)
def test_notebook(filename): nb = read(open(filename), 'json') expected = deepcopy(nb['worksheets'][0]['cells']) # TODO: figure out how to set the log-level for IPython here r = NotebookRunner(nb) r.run_notebook() actual = r.nb['worksheets'][0]['cells'] failed = [] for exp, act in zip(expected, actual): if exp['cell_type'] == 'code': #print "comparing outputs for ", exp['input'] for eo,ao in zip(exp['outputs'], act['outputs']): #print "\t", eo['text'], ao['text'], eo['text']==ao['text'] eotext = ['\n'.join(l for l in eo['text'].split('\n') if _filter_exceptions(l))] aotext = ['\n'.join(l for l in ao['text'].split('\n') if _filter_exceptions(l))] if eotext != aotext: #print "\tFAILED" failed.append({'prompt_number': exp['prompt_number'], 'input': exp['input'], 'expected_output': eotext, 'actual_output': aotext}) return len(failed)==0, failed, r.nb
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=True): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? curr_dir = os.getcwd() os.chdir(os.path.dirname(nb_path)) notebook = read(open(nb_path), 'json') nb_runner = NotebookRunner(notebook, pylab=True) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print '' print e # Return the traceback, filtering out ANSI color codes. # http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences return 'Notebook conversion failed with the following traceback: \n%s' % \ re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e)) if dest_path is None: dest_path = 'temp_evaluated.ipynb' write(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) os.chdir(curr_dir) return ret
def testRunNotebooks(self): notebook_dir = path.join('tests', 'input') for notebook_path in glob(path.join(notebook_dir, '*.ipynb')): notebook_file = path.basename(notebook_path) print(notebook_file) expected_file = path.join('tests', 'expected', notebook_file) notebook = "" with open(notebook_path) as notebook_file: notebook = notebook_file.read() try: # IPython 3 notebook = reads(notebook, 3) except (TypeError, NBFormatError): # IPython 2 notebook = reads(notebook, 'json') runner = NotebookRunner(notebook, working_dir=notebook_dir) runner.run_notebook(True) expected = "" with open(expected_file) as notebook_file: expected = notebook_file.read() try: # IPython 3 expected = reads(expected, 3) except (TypeError, NBFormatError): # IPython 2 expected = reads(expected, 'json') self.assert_notebooks_equal(expected, runner.nb)
def test_nbs(): path = os.path.join(pyfolio_root(), 'examples', '*.ipynb') for ipynb in glob.glob(path): with open(ipynb) as f: nb = read(f, 'json') nb_runner = NotebookRunner(nb) nb_runner.run_notebook(skip_exceptions=False)
def run_ipython_notebook(notebook_str): """ References: https://github.com/paulgb/runipy >>> from utool.util_ipynb import * # NOQA """ from runipy.notebook_runner import NotebookRunner import nbformat import logging log_format = '%(asctime)s %(levelname)s: %(message)s' log_datefmt = '%m/%d/%Y %I:%M:%S %p' logging.basicConfig( level=logging.INFO, format=log_format, datefmt=log_datefmt ) #fpath = 'tmp.ipynb' #notebook_str = ut.readfrom(fpath) #nb3 = IPython.nbformat.reads(notebook_str, 3) #cell = nb4.cells[1] #self = runner #runner = NotebookRunner(nb3, mpl_inline=True) print('Executing IPython notebook') nb4 = nbformat.reads(notebook_str, 4) runner = NotebookRunner(nb4) runner.run_notebook(skip_exceptions=False) run_nb = runner.nb return run_nb
def test_tutorial_notebook(): from IPython.nbformat.current import read from runipy.notebook_runner import NotebookRunner rootdir = os.path.join(aospy.__path__[0], 'examples') with open(os.path.join(rootdir, 'tutorial.ipynb')) as nb: notebook = read(nb, 'json') r = NotebookRunner(notebook) r.run_notebook() r.shutdown_kernel()
def testRunNotebooks(self): input_glob = path.join('tests', 'input', '*.ipynb') for notebook_file in glob(input_glob): notebook_file_base = path.basename(notebook_file) print notebook_file_base expected_file = path.join('tests', 'expected', notebook_file_base) runner = NotebookRunner(notebook_file) runner.run_notebook(True) expected = read(open(expected_file), 'json') self.assert_notebooks_equal(expected, runner.nb)
def testRunNotebooks(self): notebook_dir = path.join('tests', 'input') for notebook_path in glob(path.join(notebook_dir, '*.ipynb')): notebook_file = path.basename(notebook_path) print notebook_file expected_file = path.join('tests', 'expected', notebook_file) runner = NotebookRunner(read(open(notebook_path), 'json'), working_dir=notebook_dir) runner.run_notebook(True) expected = read(open(expected_file), 'json') self.assert_notebooks_equal(expected, runner.nb)
def evaluate_notebook(nb_path, dest_path=None): # Create evaluated version and save it to the dest path. nb_runner = NotebookRunner(nb_in=nb_path) nb_runner.run_notebook() if dest_path is None: dest_path = 'temp_evaluated.ipynb' nb_runner.save_notebook(dest_path) ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret
def check_ipython_notebook(): notebooks = glob.glob("*ipynb") N = len(notebooks) pb = Progress(N) for i, filename in enumerate(notebooks): print(purple(filename)) notebook = read(open(filename), 'json') r = NotebookRunner(notebook) r.run_notebook() pb.animate(i + 1)
def run(nbtxt, output=None, view=False): """ Run a notebook app 100% from JSON (text stream), return the JSON (text stream) :param nbtxt: JSON representation of notebook app, ready to run :param view: don't invoke notebook, just view in current form NOTE: `view` probably isn't useful, since the input will just be output again """ # TODO: support output parameter to specify only returning certain attributes from notebook # create a notebook object from the JSON nb_obj = nb_read_json(nbtxt) nb_runner = NotebookRunner(nb_obj) try: # get the app name from metadata name = nb_obj['metadata']['conda.app']['name'] except KeyError as ex: name = "nbapp" try: if view: pass # then don't run it else: nb_runner.run_notebook(skip_exceptions=False) return nb_runner.nb except Empty as ex: sys.stderr.write("IPython Kernel timeout") err = mini_markdown_nb(""" Notebook Error ============== ERROR: IPython Kernel timeout ``` {error} ``` """.format(error=str(ex).split(':')[-1])) return err except (NotImplementedError, NotebookError, ValueError) as ex: msg = str(ex).splitlines()[-1] sys.stderr.write(msg) err = mini_markdown_nb(""" Notebook Error ============== Notebook contains unsupported feature or bad argument: ``` {error} ``` """.format(error=msg)) return err except ImportError: msg = "nodejs or pandoc must be installed" sys.stderr.write(msg) err = mini_markdown_nb(msg) return err
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? nb_runner = NotebookRunner(nb_in=nb_path, pylab=True) nb_runner.run_notebook(skip_exceptions=skip_exceptions) if dest_path is None: dest_path = 'temp_evaluated.ipynb' nb_runner.save_notebook(dest_path) ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret
def run_notebook(nbpath): logging.info("Reading notebook '%s'", nbpath) with open(nbpath) as nbfile: notebook = nbformat.read(nbfile, as_version=3) runner = NotebookRunner(notebook) try: runner.run_notebook() except NotebookError: logging.error("An error occurred while executing notebook '%s'. " "Exiting with nonzero exit status", nbpath) sys.exit(1)
def run_notebook(nbpath): logging.info("Reading notebook '%s'", nbpath) with open(nbpath) as nbfile: notebook = read(nbfile, 'json') runner = NotebookRunner(notebook, mpl_inline=True) try: runner.run_notebook() except NotebookError: logging.error( "An error occurred while executing notebook '%s'. " "Exiting with nonzero exit status", nbpath) sys.exit(1)
def evaluate_notebook(nb_path, dest_path=None): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? nb_json = nbr(open(nb_path), 'json') nb_runner = NotebookRunner(nb_json, pylab=True) nb_runner.run_notebook() if dest_path is None: dest_path = 'temp_evaluated.ipynb' #nb_runner.save_notebook(dest_path) nbw(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret
def test_nbs(): path = os.path.join(pyfolio_root(), 'examples', '*.ipynb') for ipynb in glob.glob(path): # See if bayesian is useable before we run a test if ipynb.endswith('bayesian.ipynb'): try: import pymc3 # NOQA except: continue with open(ipynb) as f: nb = read_notebook(f, 'json') nb_runner = NotebookRunner(nb) nb_runner.run_notebook(skip_exceptions=False)
def collect(self): with self.fspath.open() as f: payload = f.read() self.nb = nbformat.reads(payload, 3) # kernel needs to start from the same dir the ipynb is in notebook_dir = self.fspath.dirname cwd = os.getcwd() if cwd != notebook_dir: os.chdir(notebook_dir) self.runner = NotebookRunner(self.nb) os.chdir(cwd) cell_num = 1 for cell in self.runner.iter_code_cells(): yield IPyNbCell.from_parent(self, name=self.name, cell_num=cell_num, cell=cell) cell_num += 1
def test(self): payload = open(filename) nb = read(payload, 'json') # turn off colors, so we can properly read exceptions cell_nc = new_code_cell(input='%colors NoColor', prompt_number=0) nb.worksheets[0].cells.insert(0, cell_nc) # workaround for matplotlib backend cell_mpl = new_code_cell(input="import matplotlib; matplotlib.use('Agg')", prompt_number=1) nb.worksheets[0].cells.insert(1, cell_mpl) # set working dir to notebook path wd = os.path.abspath(os.path.dirname(filename)) runner = NotebookRunner(nb, working_dir=wd) #try: runner.run_notebook(skip_exceptions=False)
def run(self): # Now convert the lecture notes, problem sets, and practice problems to # HTML notebooks. from runipy.notebook_runner import NotebookRunner start_dir = os.path.abspath('.') for notebook in glob.glob('*.ipynb'): if os.path.dirname(notebook): os.chdir(os.path.dirname(notebook)) r = NotebookRunner("./" + os.path.basename(notebook)) r.run_notebook(skip_exceptions=True) r.save_notebook("./" + os.path.basename(notebook)) os.chdir(start_dir)
def collect(self): with self.fspath.open() as f: payload = f.read() self.notebook_folder = self.fspath.dirname try: # Ipython 3 self.nb = reads(payload, 3) except (TypeError, NBFormatError): # Ipython 2 self.nb = reads(payload, 'json') self.runner = NotebookRunner(self.nb) cell_num = 1 for cell in self.runner.iter_code_cells(): yield IPyNbCell(self.name, self, cell_num, cell) cell_num += 1
def execute_notebook(nb, resources): """Execute notebook. With ExecutePreprocessor using IPython >= 3 or runipy instead. """ if is_ipython_3(): from IPython.nbconvert.preprocessors import ExecutePreprocessor nb, resources = ExecutePreprocessor().preprocess(nb, resources) elif runipy_available: from runipy.notebook_runner import NotebookRunner r = NotebookRunner(nb) r.run_notebook(skip_exceptions=True) nb = r.nb else: raise ImportError( "Can't execute notebooks. Please install IPython >= 3 or runipy.") return nb
def render_nb(): # need ipynb v3 to play nice with runipy notebook = read(open("stock_infos.ipynb"), 3) nb = NotebookRunner(notebook, pylab=True) nb.run_notebook() # need ipynb v4 to play nice with Jupyter nb = nbformat.convert(nb.nb, 4) html_exporter = HTMLExporter() body, resources = html_exporter.from_notebook_node(nb) html_file = open("static/stock_infos.html", "w") html_file.write(body.encode('utf8', 'ignore')) html_file.close() return app.send_static_file('stock_infos.html')
def run(self): # Now convert the lecture notes, problem sets, and practice problems to # HTML notebooks. from runipy.notebook_runner import NotebookRunner from IPython.nbformat.current import read, write start_dir = os.path.abspath('.') for notebook in glob.glob('notebooks/*.ipynb'): print("Running {0}...".format(notebook)) os.chdir(os.path.dirname(notebook)) with open(os.path.basename(notebook)) as f: r = NotebookRunner(read(f, 'json'), pylab=False) r.run_notebook(skip_exceptions=True) with open(os.path.basename(notebook), 'w') as f: write(r.nb, f, 'json') os.chdir(start_dir)
def run(self): # Now convert the lecture notes, problem sets, and practice problems to # HTML notebooks. from runipy.notebook_runner import NotebookRunner start_dir = os.path.abspath('.') for notebook in (glob.glob('lectures/*.ipynb') + glob.glob('problems/*.ipynb') + glob.glob('practice/*.ipynb')): if "Understanding" in notebook: continue os.chdir(os.path.dirname(notebook)) r = NotebookRunner(os.path.basename(notebook), pylab=True) r.run_notebook(skip_exceptions=True) r.save_notebook(os.path.basename(notebook)) os.chdir(start_dir)
def notebooks(env, loader, args): nb_path = env.resolve('notebooks') res_path = env.resolve('results') for filename in os.listdir(nb_path): if filename.endswith('.ipynb'): nb_name = filename.split('.')[0] print('running', nb_name) src_path = os.path.join(nb_path, filename) html_path = os.path.join(res_path, nb_name + '.html') with open(src_path, 'r') as f: notebook = read(f, 'json') r = NotebookRunner(notebook, working_dir=nb_path) r.run_notebook() dest_path = tempfile.mkstemp() t = tempfile.NamedTemporaryFile() with open(t.name, 'w') as g: write(r.nb, g) exporter = nbconvert.HTMLExporter() body, resources = exporter.from_filename(t.name) with open(html_path, 'wb') as g: g.write(body.encode('utf-8'))
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False): # Create evaluated version and save it to the dest path. # Always use --pylab so figures appear inline # perhaps this is questionable? notebook = read(open(nb_path), 'json') nb_runner = NotebookRunner(notebook, pylab=True, mpl_inline=True) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print('\n', '-' * 80) print(e) print('-' * 80) raise if dest_path is None: dest_path = 'temp_evaluated.ipynb' write(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False): # Create evaluated version and save it to the dest path. notebook = nbformat.read(open(nb_path), 'json') nb_runner = NotebookRunner(notebook, pylab=False) try: nb_runner.run_notebook(skip_exceptions=skip_exceptions) except NotebookError as e: print('') print(e) # Return the traceback, filtering out ANSI color codes. # http://stackoverflow.com/questions/13506033/filtering-out-ansi-escape-sequences return "Notebook conversion failed with the " \ "following traceback: \n%s" % \ re.sub(r'\\033[\[\]]([0-9]{1,2}([;@][0-9]{0,2})*)*[mKP]?', '', str(e)) if dest_path is None: dest_path = 'temp_evaluated.ipynb' nbformat.write(nb_runner.nb, open(dest_path, 'w'), 'json') ret = nb_to_html(dest_path) if dest_path is 'temp_evaluated.ipynb': os.remove(dest_path) return ret