def run_notebook(path): nb_name, _ = os.path.splitext(os.path.basename(path)) dirname = os.path.dirname(path) with open(path) as f: nb = nbformat.read(f, as_version=4) print("Start ", path) sys.stdout.flush() proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True # For Python >= 3.8 on Windows, the Preprocessor throws a NotImplementedError # This check is inserted as a workaround # See: https://github.com/jupyter/nbconvert/issues/1372 if sys.version_info[0] == 3 and sys.version_info[ 1] >= 8 and sys.platform.startswith('win'): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) proc.preprocess(nb, {'metadata': {'path': dirname}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) if errors == []: print(" " + path + " test successfully completed.") return 0 else: print(" " + path + " test exited with errors.") return 1
def run_notebook(notebook_path): # https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with Path(dirname): # run in notebook fir? with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=60, kernel_name="deep_ml_curriculum") proc.allow_errors = False proc.preprocess(nb, {"metadata": {"path": "/"}}) output_path = os.path.join(dirname, "{}_all_output.ipynb".format(nb_name)) with open(output_path, mode="wt") as f: nbformat.write(nb, f) errors = [] for i, cell in enumerate(nb.cells): if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": logging.error("nb %r, cell %s, ename %r, error %r", nb_name, i, output['ename'], output['evalue']) errors.append(output) return nb, errors
def run_notebook(notebook_path, cleanup=True): """Helper script to run notebooks. Adapted with minor modifications from https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ """ nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) output_path = os.path.join(dirname, f'{nb_name}_all_output.ipynb') # temp file is created so as not to mess up the original notebook with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) if cleanup: os.remove(output_path) return nb, errors
def run_notebook(notebook_path): """ A function to run every cell in a Jupyter notebook. Opens, runs, and saves the Jupyter notebook. Notebook saved with the same filename. Function will overwrite the notebook it runs. Usage from nbtools import run_notebook run_notebook('Untitled.ipynb') :param notebook_path: path to notebook, needs to be readable by os.path :return: None, but a notebook (the one that is run) is saved to disk """ nb_name = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernal_name="python3") proc.allow_errors = True proc.preprocess(nb, {"metadata": {"path": "/"}}) with open(notebook_path, mode="wt") as f: nbformat.write(nb, f)
def run_notebook(notebook_path, iam_apikey, wksp_id, test_file, output_path): """ Run notebook for end to end test :param notebook_path: :param uname: :param pwd: :param wksp_id: :param test_file: :param output_path: """ notebook_name, _ = os.path.splitext(os.path.basename(notebook_path)) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) nb, old_cred_text = _replace_nb_input(nb, iam_apikey, wksp_id, test_file) # nb = _remove_experimentation(nb) proc = ExecutePreprocessor(timeout=60 * 60, kernel_name="python3") proc.allow_errors = True proc.preprocess(nb, {"metadata": {"path": os.getcwd()}}) errors = [] for cell in nb.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output) if "source" in cell and "iam_apikey = " in cell["source"]: cell["source"] = old_cred_text with open(output_path + ".ipynb", mode="wt") as f: nbformat.write(nb, f) return nb, errors
def test_run(notebook_path): """Adapted from: https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/""" kernel_name = get_a_jupyter_kernel_name() output_path = tempfile.NamedTemporaryFile().name nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def test_notebooks(): # define process executor with timeout in seconds and python runtime proc = ExecutePreprocessor(timeout=60, kernel_name='python3') proc.allow_errors = True # test all Jupyte notebooks from folder (remove hidden folders) for i in range(len(FOLDERS)): # include only not hidden jupyter notebooks files = sorted([y for x in os.walk(FOLDERS[i]) for y in glob(os.path.join(x[0], '[!.]*.ipynb'))]) print() print('Parsing files from ' + FOLDERS[i] + ' with ' + str(len(files)) + ' files ') print('+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-') errors = [] nb = None i = 1 for file in files: print(str(i) + ' - Testing notebook ' + file) with open(file) as f: nb = nbformat.read(f, as_version=4) proc.preprocess(nb, {'metadata': {'path': '/'}}) for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append({'notebook': file, 'cell': cell.execution_count, 'trace': output}) i = i + 1 return nb, errors
def run_notebook(notebook_path, working_directory='/', kernel=os.getenv("TOX_KERNEL") or "python3"): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, iopub_timeout=10, kernel_name=kernel) proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': working_directory}}) output_path = os.path.join(dirname, '{}.outnb'.format(nb_name)) with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(path): nb_name, _ = os.path.splitext(os.path.basename(path)) dirname = os.path.dirname(path) with open(path) as f: nb = nbformat.read(f, as_version=4) print("Start ", path) sys.stdout.flush() proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': dirname}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) if errors == []: print(" " + path + " test successfully completed.") return 0 else: print(" " + path + " test exited with errors.") return 1
def run_notebook(notebook_path, kernel_name=None): """ From: http://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ """ nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) output_path = os.path.join(dirname, '{}_all_output.ipynb'.format(nb_name)) with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(notebook_path, iam_apikey, wksp_id, test_file, output_path): ''' Run notebook for end to end test :param notebook_path: :param uname: :param pwd: :param wksp_id: :param test_file: :param output_path: ''' notebook_name, _ = os.path.splitext(os.path.basename(notebook_path)) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) nb, old_cred_text = _replace_nb_input(nb, iam_apikey, wksp_id, test_file) #nb = _remove_experimentation(nb) proc = ExecutePreprocessor(timeout=60 * 60, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': os.getcwd()}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) if 'source' in cell and 'iam_apikey = ' in cell['source']: cell['source'] = old_cred_text with open(output_path + '.ipynb', mode='wt') as f: nbformat.write(nb, f) return nb, errors
def run_notebook(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) nb['cells'] = [ cell for cell in nb['cells'] if not cell['source'].startswith('!') ] proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': 'examples/'}}) output_path = os.path.join(dirname, '_test_{}.ipynb'.format(nb_name)) with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(notebook_path): with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name="python3") proc.allow_errors = True script_path = os.path.dirname(os.path.abspath(__file__)) package_path = os.path.abspath(os.path.join(script_path, "..", "..")) # Add shutdown for show method. shutdown_cell = new_code_cell( "from interpret import shutdown_show_server\nshutdown_show_server()") nb.cells.append(shutdown_cell) # In the second cell, enable sampling nb.cells[1]["source"] = re.sub("# df = df.sample", "df = df.sample", nb.cells[1]["source"]) proc.preprocess(nb, {"metadata": {"path": package_path}}) errors = [] for cell in nb.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output) return nb, errors
def run_notebook(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name="python3") proc.allow_errors = True proc.preprocess(nb, {"metadata": {"path": "/"}}) output_path = os.path.join(dirname, "{}_all_output.ipynb".format(nb_name)) with open(output_path, mode="wt") as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output) ## Remove temporary file os.remove(output_path) return nb, errors
def _notebook_run(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3',allowerrors=True) proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': dirname}}) #critically the notebooks are setup to run from their local directory so this needs to be specified output_path = os.path.join(dirname, '{}_all_output.ipynb'.format(nb_name)) with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = {} for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors[cell['execution_count']]=output os.remove(output_path) return nb, errors
def setUpClass(cls): notebook_path = os.path.join(os.path.dirname(__file__), 'testnb.ipynb') with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600) proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) cls.cells = nb.cells
def run_jn(jn, timeout): open_jn = open(jn, "r", encoding='utf-8') notebook = nbformat.read(open_jn, nbformat.current_nbformat) open_jn.close() preprocessor = ExecutePreprocessor(timeout=timeout, kernel_name='python3') preprocessor.allow_errors = True preprocessor.preprocess(notebook, {'metadata': {'path': os.path.dirname(jn)}}) return notebook
def test_files(nb_file): kernel_name = os.environ.get("NOTEBOOK_KERNEL", "python3") with open(nb_file) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) proc.allow_errors = True proc.preprocess(nb, {"metadata": {"path": "/"}}) cells_with_outputs = [c for c in nb.cells if "outputs" in c] for cell in cells_with_outputs: for output in cell["outputs"]: if output.output_type == "error": for l in output.traceback: print(l) raise Exception(f"{output.ename}: {output.evalue}")
def _run_notebook(dir_path, notebook_path): with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(kernel_name="python3") proc.allow_errors = True proc.preprocess(nb, {"metadata": {"path": dir_path}}) return [ output for cell in nb.cells for output in cell.get("outputs", []) if output.output_type == "error" ]
def run_notebook(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) output_path = os.path.join(dirname, '{}_all_output.ipynb'.format(nb_name)) with open(output_path, mode='wt') as f: nbformat.write(nb, f)
def run_notebook(notebook_name): with open(notebook_name) as f: nb = nbformat.read(f, as_version=nbformat.NO_CONVERT) ep = ExecutePreprocessor(timeout=100) ep.allow_errors = True ep.preprocess(nb, {'metadata': {'path': os.path.dirname(notebook_name)}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return errors
def run_notebook(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb) for num, cell in enumerate(nb.cells): if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': return cell.execution_count, output.traceback return None
def get_errors(path): with open(path) as f: notebook = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=None, kernel_name='python3') proc.allow_errors = True proc.preprocess(notebook, {'metadata': {'path': '/'}}) errors = [] for cell in notebook.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return errors
def run_notebook(notebook_path): _logger.info(f"Running {notebook_path}") with open(notebook_path, encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(notebook_path): with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True script_path = os.path.dirname(os.path.abspath(__file__)) package_path = os.path.abspath(os.path.join(script_path, '..', '..')) proc.preprocess(nb, {'metadata': {'path': package_path}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(notebook_path): with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name="python3") proc.allow_errors = True script_path = os.path.dirname(os.path.abspath(__file__)) package_path = os.path.abspath(os.path.join(script_path, "..", "..")) proc.preprocess(nb, {"metadata": {"path": package_path}}) errors = [] for cell in nb.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output) return nb, errors
def run_notebook(nbpath, kernel='python3'): """Execute notebook with nbconvert, save to a new file, and parse for errors. Parameters ---------- nbpath : str Path to the .ipynb notebook file to be tested. kernel : str Name of kernel to be used. Default: python3 Currently kernel switching is not yet implemented. Attribution ----------- Some code in this function from http://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ """ nbname, _ = os.path.splitext(os.path.basename(nbpath)) dirname = os.path.dirname(nbpath) with open(nbpath) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=6000, kernel_name=kernel) proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': dirname}}) output_path = os.path.join(dirname, '{}_tested.ipynb'.format(nbname)) with open(output_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nbpath, nb, errors
def generate_html(self, kernel_name=""): nb_executor = ExecutePreprocessor(kernel_name=kernel_name) nb_executor.allow_errors = True html_exporter = HTMLExporter() html_writer = FilesWriter() with open(self.output_ipynb_path) as f: nb = nbformat.read(f, as_version=4) nb = nb_executor.preprocess(nb)[0] body, resources = html_exporter.from_notebook_node(nb) # HTML writer automatically adds .html to the end, so get rid of it html_writer.write( body, resources, self.output_html_path.replace(".html", "") ) print(f"Written file to {self.output_html_path}") if self.verbose else None self.metadata["html-file"] = os.path.basename(self.output_html_path) self.metadata.mark_as_updated()
def run_notebook(notebook_path: str) -> Tuple[Any, List[Any]]: full_path = os.path.join(ROOT_DIR, notebook_path) nb_name, _ = os.path.splitext(os.path.basename(full_path)) with open(full_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': os.path.dirname(full_path)}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': '/'}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
for fname in sorted(glob.glob1(src_dir, '*.ipynb')): basename = fname[:-6] output_ipynb_fname = os.path.join(target_dir, fname) output_rst_fname = os.path.join(target_dir, basename + '.rst') print 'Running', fname with open(os.path.join(src_dir, fname), 'r') as f: notebook = reads(f.read()) # The first line of the tutorial file should give the title title = notebook.cells[0]['source'].split('\n')[0].strip('# ') tutorials.append((basename, title)) # Execute the notebook preprocessor = ExecutePreprocessor() preprocessor.allow_errors = True notebook, _ = preprocessor.preprocess(notebook, {'metadata': {'path': src_dir}}) print 'Saving notebook and converting to RST' exporter = NotebookExporter() output, _ = exporter.from_notebook_node(notebook) with codecs.open(output_ipynb_fname, 'w', encoding='utf-8') as f: f.write(output) # Insert a note about ipython notebooks with a download link note = deindent(u''' .. only:: html .. |launchbinder| image:: http://mybinder.org/badge.svg .. _launchbinder: http://mybinder.org:/repo/brian-team/brian2-binder/notebooks/tutorials/{tutorial}.ipynb