def run_notebook(path, timeout=120): """ Execute a Jupyter Notebook and return any errors that it produces. :param path: path to the .ipynb file to execute :param int timeout: number of seconds to let the notebook run before we throw an exception by default. :return: a tuple (nb, errors) containing the parsed Notebook object and a list of errors, respectively """ # We'll use a NotebookExporter from the nbconvert package to load the notebook and re-export it to a temporary file. # First we want to configure it to execute the notebook before writing it: c = Config() c.NotebookExporter.preprocessors = ['nbconvert.preprocessors.ExecutePreprocessor'] c.ExecutePreprocessor.timeout = timeout c.ExecutePreprocessor.kernel_name = 'LEAP_venv' # We assume a kernel named "LEAP_venv" that lives in our venv exp = NotebookExporter(config=c) # Load the notebook with open(path, 'r') as nb_file: body, resources = exp.from_file(nb_file) # Parse the notebook string into a notebook object nb = nbformat.reads(body, nbformat.current_nbformat) errors = [output for cell in nb.cells if "outputs" in cell for output in cell["outputs"] if output.output_type == "error"] return nb, errors
def _clear_notebook(self, node): # pragma: no cover if NotebookExporter is not None: exporter = NotebookExporter() exporter.register_preprocessor(ClearOutputPreprocessor(enabled=True)) cleared,_ = exporter.from_notebook_node(node) else: stripped_node = v3_strip_output(node) cleared = current.writes(stripped_node, 'ipynb') return cleared
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False, skip_execute=None, timeout=300, ipython_startup=None, patterns_to_take_with_me=None): if patterns_to_take_with_me is None: patterns_to_take_with_me = [] notebook = nbformat.read(nb_path, as_version=4) kwargs = dict(timeout=timeout, kernel_name='python%s' % sys.version_info[0], allow_errors=skip_exceptions) cwd = os.getcwd() filedir, filename = os.path.split(nb_path) os.chdir(filedir) not_nb_runner = ExecutePreprocessor1000(**kwargs) if ipython_startup is not None: not_nb_runner._ipython_startup = ipython_startup if not os.path.isfile(dest_path): print('INFO: Writing evaluated notebook to {dest_path!s}'.format( dest_path=os.path.abspath(dest_path))) try: if not skip_execute: not_nb_runner.preprocess(notebook, {}) except CellExecutionError as e: print('') print(e) os.chdir(cwd) if skip_execute: nbformat.write(notebook, open(dest_path, 'w')) else: ne = NotebookExporter() newnb, _ = ne.from_notebook_node(notebook) with open(dest_path, 'w') as f: f.write(newnb) for pattern in patterns_to_take_with_me: for f in glob.glob( os.path.join(os.path.dirname(nb_path), pattern)): print("mv %s %s" % (f, os.path.dirname(dest_path))) shutil.move(f, os.path.dirname(dest_path)) else: print( 'INFO: Skipping existing evaluated notebook {dest_path!s}'.format( dest_path=os.path.abspath(dest_path)))
def render_track(track, track_cfg): meta = utils.get_track_meta(track, track_cfg) cfg = Config() cfg.Exporter.preprocessors = ['nb_utils.lesson_preprocessor.LearnLessonPreprocessor'] exporter = NotebookExporter(config=cfg) resources = {'track_meta': meta, 'track_cfg': track_cfg} outdir = os.path.join(track, track_cfg['tag'], 'rendered') os.makedirs(outdir, exist_ok=True) for nb_meta in meta.notebooks: in_path = os.path.join(track, 'raw', nb_meta.filename) resources['lesson'] = nb_meta.lesson resources['nb_meta'] = nb_meta if CLEAN: clean(in_path) nb, _ = exporter.from_filename(in_path, resources) out_path = os.path.join(outdir, nb_meta.filename) with open(out_path, 'w') as f: f.write(nb)
def clean(): _delete_generated_files() config = Config() config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()] exporter = NotebookExporter(config=config) for notebook in get_files("ipynb"): with open(notebook, encoding="utf8") as nb_file: nb = nbformat.read(nb_file, as_version=4) if not nb.cells[-1]["source"]: nb.cells.pop() format_code_cells(nb) for cell_id, cell in enumerate(nb.cells): cell["id"] = f"{notebook.stem}-{cell_id}" ipynb, _ = exporter.from_notebook_node(nb) with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable: writable.write(ipynb)
def render_track(track, nb_path_whitelist=None): meta = utils.get_track_meta(track) track_cfg = utils.get_track_config(track) cfg = Config() cfg.Exporter.preprocessors = [ 'lesson_preprocessor.LearnLessonPreprocessor' ] exporter = NotebookExporter(config=cfg) resources = {'track_meta': meta, 'track_cfg': track_cfg} for nb_meta in meta.notebooks: in_path = os.path.join(track, 'raw', nb_meta.filename) if nb_path_whitelist and in_path not in nb_path_whitelist: continue resources['lesson'] = nb_meta.lesson resources['nb_meta'] = nb_meta if CLEAN: clean(in_path) nb, _ = exporter.from_filename(in_path, resources) out_path = os.path.join(track, 'rendered', nb_meta.filename) with open(out_path, 'w') as f: f.write(nb)
def main(): nb = nbformat.read('04-merge-pivot-answers.ipynb', as_version=nbformat.NO_CONVERT) # In[5]: exporter = NotebookExporter() body, resources = exporter.from_notebook_node(nb) # In[6]: c = Config() c.NotebookExporter.preprocessors = [ExercisePreprocessor] exercise_exporter = NotebookExporter(config=c) # exercise_exporter.preprocessors # In[7]: body2, res2 = exercise_exporter.from_notebook_node(nb) # In[8]: nb2 = nbformat.reads(body2, 4)
'CachedOutputPreprocessor': { 'enabled': True, 'cache_directory': '.nb_output_cache', 'req_files': [ 'code/init_mooc_nb.py', 'code/edx_components.py', 'code/pfaffian.py', 'code/functions.py' ], 'warnings': 'ignore', 'timeout': 300 } }) cachedoutput = NotebookExporter(cfg) with open('scripts/release_dates') as f: release_dates = eval(f.read()) def date_to_edx(date, add_days=0): tmp = strptime(date, '%d %b %Y') date = datetime.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, 10) date = date + datetime.timedelta(days=add_days) date = date.strftime('%Y-%m-%dT%H:%M:%SZ') return date def parse_syllabus(syllabus_file, content_folder=''):
preprocessor_list.append(FormatCodeCellPreprocessor) if execute_code: preprocessor_list.append(preprocessors.ExecutePreprocessor) # Create the exporters with preprocessing c = Config() c.NotebookExporter.preprocessors = preprocessor_list c.HTMLExporter.preprocessors = preprocessor_list if execute_code: c.ExecutePreprocessor.timeout = cell_timeout if execute_code != "default": c.ExecutePreprocessor.kernel_name = execute_code nb_exporter = NotebookExporter(c) html_exporter = HTMLExporter(c) # html_exporter.template_file = 'basic' # Find all Jupyter notebook files in the specified directory all_files = [] for p in args.locations: path = Path(p) if path.is_dir(): all_files.extend(path.glob("**/*.ipynb")) elif path.is_file(): all_files.append(path) else: raise ValueError( f"Specified location not '{path}'a file or directory.")
preprocessor_list.append(ClearWarningsPreprocessor) if coalesce_streams: preprocessor_list.append(preprocessors.coalesce_streams) # Create the exporters with preprocessing c = Config() c.NotebookExporter.preprocessors = preprocessor_list c.HTMLExporter.preprocessors = preprocessor_list if execute_code: c.ExecutePreprocessor.timeout = cell_timeout if execute_code != "default": c.ExecutePreprocessor.kernel_name = execute_code nb_exporter = NotebookExporter(c) html_exporter = HTMLExporter(c) # html_exporter.template_file = 'basic' # Find all Jupyter notebook files in the specified directory all_files = [] for p in args.locations: path = Path(p) if path.is_dir(): all_files.extend(path.glob("**/*.ipynb")) elif path.is_file(): all_files.append(path) else: raise ValueError( f"Specified location not '{path}'a file or directory.")
c = Config() c.NotebookExporter.preprocessors = [ "nbconvert.preprocessors.RegexRemovePreprocessor", "nbconvert.preprocessors.ExecutePreprocessor", ] # Remove any cells containing lines with the phrase "Table of Contents" or # consist solely of whitespace c.RegexRemovePreprocessor.enabled = True c.RegexRemovePreprocessor.patterns = [".*Table of Contents.*", "\\s*\\Z"] # Execute each notebook in a python3 kernel to make sure the content is there c.ExecutePreprocessor.enabled = True c.ExecutePreprocessor.kernel = "python3" exporter = NotebookExporter(config=c) for nb in os.listdir(os.path.join(ROOT, "examples")): # Skip any non-notebooks if nb.endswith("ipynb") and "no-doc" not in nb: nb_in = os.path.join(ROOT, "examples", nb) nb_out = os.path.join(ROOT, "docs/source/read-only-examples", nb) # Skip any notebooks which already exist if not os.path.exists(nb_out): with open(nb_out, "w") as out_stream: converted = exporter.from_filename(nb_in)[0] out_stream.write(converted) import skcosmo # noqa
def evaluate_notebook(nb_path, dest_path=None, skip_exceptions=False, substring=None, end=None, skip_execute=None, skip_output=None, offset=0, timeout=300, ipython_startup=None, patterns_to_take_with_me=None): if patterns_to_take_with_me is None: patterns_to_take_with_me = [] notebook = nbformat.read(nb_path, as_version=4) kwargs = dict(timeout=timeout, kernel_name='python%s' % sys.version_info[0], allow_errors=skip_exceptions) cwd = os.getcwd() filedir, filename = os.path.split(nb_path) os.chdir(filedir) #profile_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'profile_docs') #print("Ignoring %s...hope that's ok"%profile_dir) not_nb_runner = ExecutePreprocessor1000(**kwargs) if ipython_startup is not None: not_nb_runner._ipython_startup = ipython_startup if not os.path.isfile(dest_path): # TODO but this isn't true, is it? it's running the originl nb print('INFO: Writing evaluated notebook to {dest_path!s}'.format( dest_path=os.path.abspath(dest_path))) try: if not skip_execute: not_nb_runner.preprocess(notebook, {}) except CellExecutionError as e: print('') print(e) os.chdir(cwd) if skip_execute: nbformat.write(notebook, open(dest_path, 'w')) else: ne = NotebookExporter() newnb, _ = ne.from_notebook_node(notebook) with open(dest_path, 'w') as f: f.write(newnb) for pattern in patterns_to_take_with_me: for f in glob.glob( os.path.join(os.path.dirname(nb_path), pattern)): print("mv %s %s" % (f, os.path.dirname(dest_path))) shutil.move(f, os.path.dirname(dest_path)) else: print( 'INFO: Skipping existing evaluated notebook {dest_path!s}'.format( dest_path=os.path.abspath(dest_path))) preprocessors = [] if substring is None and not offset else [ NotebookSlice(substring, end, offset) ] preprocessors = ( preprocessors + [SkipOutput(skip_output)]) if skip_output else preprocessors ret = nb_to_html(dest_path, preprocessors=preprocessors) return ret
def to_notebook(self): return NotebookExporter().from_notebook_node(self._content)[0]
class CleanTagReplacer(Preprocessor): """Replace cells tagged 'clean' with a dummy cell""" def preprocess( self, nb: "NotebookNode", resources: Dict[Any, Any]) -> Tuple["NotebookNode", Dict[Any, Any]]: """Preprocess the entire notebook.""" for i, cell in enumerate(nb.cells): if "tags" in cell.metadata and "clean" in cell.metadata["tags"]: nb.cells[i] = new_code_cell( source="# Write Python source code in this cell") return nb, resources nb_exp = NotebookExporter(preprocessors=[CleanTagReplacer]) if __name__ == "__main__": HERE = Path(__file__).parent output_directory = HERE.joinpath("output") for notebook in HERE.glob("*.ipynb"): output_notebook = notebook.with_stem(notebook.stem + "-clean").name nb = nbformat.read(notebook, as_version=4) nb_new = upgrade(nb) if nb_new is not None: nb = nb_new if "celltoolbar" in nb.metadata: del nb.metadata["celltoolbar"] clean_nb, _ = nb_exp.from_notebook_node(nb)