def test_tutorial_nb(file_path): """Run tutorial jupyter notebook to catch any execution error. Parameters ---------- file_path : str path of tutorial markdown file """ tutorial_name = os.path.basename(file_path) notebook = nbformat.read(file_path + '.ipynb', as_version=4) eprocessor = ExecutePreprocessor(timeout=1800) try: eprocessor.preprocess(notebook, {'metadata': {}}) except Exception as err: err_msg = str(err) fail_dict[tutorial_name] = err_msg finally: output_nb = open("output.txt", mode='w') nbformat.write(notebook, output_nb) output_nb.close() output_nb = open("output.txt", mode='r') for line in output_nb: if "Warning:" in line: fail_dict[tutorial_name] = "%s has warning." % (tutorial_name) return
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''): """ Execute notebook in `src` and write the output to `dst` Parameters ---------- src, dst: str path to notebook allow_errors: bool timeout: int kernel_name: str defualts to value set in notebook metadata Returns ------- dst: str """ import nbformat from nbconvert.preprocessors import ExecutePreprocessor with io.open(src, encoding='utf-8') as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, resources={}) with io.open(dst, 'wt', encoding='utf-8') as f: nbformat.write(nb, f) return dst
def _notebook_run(path): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ kernel_name = "python%d" % sys.version_info[0] this_file_directory = os.path.dirname(__file__) errors = [] with tempfile.NamedTemporaryFile(suffix=".ipynb", mode="wt") as fout: with open(path) as f: nb = nbformat.read(f, as_version=4) nb.metadata.get("kernelspec", {})["name"] = kernel_name ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10) try: ep.preprocess(nb, {"metadata": {"path": this_file_directory}}) except CellExecutionError as e: if "SKIP" in e.traceback: print(str(e.traceback).split("\n")[-2]) else: raise e except TimeoutError as e: print(e) finally: nbformat.write(nb, fout) # nb = nbformat.read(fout, nbformat.current_nbformat) # errors = errors.extend( # [output for cell in nb.cells if "outputs" in cell # for output in cell["outputs"] if output.output_type == "error"]) return nb, errors
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None): """ Execute notebook in `src` and write the output to `dst` Parameters ---------- src, dst: str path to notebook allow_errors: bool timeout: int kernel_name: str defualts to value set in notebook metadata Returns ------- dst: str """ with io.open(src, encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, {"metadta": {"path": "notebooks/"}}) with io.open(dst, "wt", encoding="utf-8") as f: nbformat.write(nb, f) return dst
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'): ''' This functions executes a Jupyter notebook and creates the related HTML file. Args: root (str): name of the file without the .ipynb extension template (str): name of the template (to be in the current folder as template.tpl) version (int): version of the notebook timeout (float): maximum time spent per cell kernel (str) Returns: None The function executes root.ipynb into root_exe.ipynb and creates the file root.html. ''' with open(root + '.ipynb') as f: nb = nbformat.read(f, as_version=version) ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) ep.preprocess(nb, {'metadata': {'path': '.'}}) with open(root + '_exe.ipynb', 'wt') as f: nbformat.write(nb, f) html_exporter = HTMLExporter() html_exporter.template_file = template with open(root + '_exe.ipynb', mode='r') as f: notebook = nbformat.reads(''.join(f.readlines()), as_version=version) (body, _) = html_exporter.from_notebook_node(notebook) codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def run_notebook(notebook_name, nb_kwargs=None, insert_pos=5, timeout=120, execute_kwargs=None): """Runs a notebook and displays the output in the master notebook. Executes a notebook, optionally passing "arguments" in a way roughly similar to passing arguments to a function. Notebook arguments are passed in a dictionary (`nb_kwargs`) which is converted to a string containing python code, then inserted in the notebook as a code cell. The code contains only assignments of variables which can be used to control the execution of a suitably written notebook. When calling a notebook, you need to know which arguments (variables) to pass. Differently from functions, no check on the input arguments is performed. Arguments: notebook_name (string): name of the notebook to be executed. nb_kwargs (dict or None): If not None, this dict is converted to a string of python assignments with keys representing variables names and values variables content. This string is inserted as code-cell in the notebook to be executed. insert_pos (int): position of insertion of the code-cell containing the input arguments. Default is 5 (i.e. sixth cell). With this default, the input notebook can define, in the first cell, default values of input arguments (used when the notebook is executed with no arguments or through the Notebook GUI). timeout (int): timeout in seconds after which the execution is aborted. execute_kwargs (dict): additional arguments passed to `ExecutePreprocessor`. """ if nb_kwargs is not None: header = '# Cell inserted during automated execution.' code = dict_to_code(nb_kwargs) code_cell = '\n'.join((header, code)) if execute_kwargs is None: execute_kwargs = {} ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs) nb = nbformat.read(notebook_name, as_version=4) if len(nb_kwargs) > 0: nb['cells'].insert(insert_pos, nbformat.v4.new_code_cell(code_cell)) try: # Execute the notebook ep.preprocess(nb, {'metadata': {'path': './'}}) #***************************************** # Title: analysis.ipynb # Author: Benjamin RK # Date: April 2013 # Availability: https://gist.github.com/minrk/5491090 ip = get_ipython() for cell in nb.cells: if cell.cell_type != 'code': continue ip.run_cell(cell.source) #***************************************** except: # Execution failed, print a message then raise. msg = 'Error executing the notebook "%s".\n\n' % notebook_name print(msg) raise
def _notebook_run(path): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ kernel_name = 'python%d' % sys.version_info[0] this_file_directory = os.path.dirname(__file__) errors = [] with tempfile.NamedTemporaryFile(suffix=".ipynb", mode='wt') as fout: with open(path) as f: nb = nbformat.read(f, as_version=4) nb.metadata.get('kernelspec', {})['name'] = kernel_name ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10) try: ep.preprocess(nb, {'metadata': {'path': this_file_directory}}) except CellExecutionError as e: if "SKIP" in e.traceback: print(str(e.traceback).split("\n")[-2]) else: raise e except RuntimeError as e: print(e) finally: nbformat.write(nb, fout) return nb, errors
def execute(self): print("Cleaning lowfat/reports/html ...") old_reports = os.listdir("lowfat/reports/html") for old_report in old_reports: print("- Removing lowfat/reports/html/{}".format(old_report)) os.remove("lowfat/reports/html/{}".format(old_report)) print("Cleaning of lowfat/reports/html is complete.") notebook_filenames = os.listdir("lowfat/reports") for notebook_filename in notebook_filenames: if not notebook_filename.endswith(".ipynb"): continue print("Processing lowfat/reports/{}".format(notebook_filename)) # Based on Executing notebooks, nbconvert Documentation by Jupyter Development Team. # https://nbconvert.readthedocs.io/en/latest/execute_api.html with open("lowfat/reports/{}".format(notebook_filename)) as file_: notebook = nbformat.read(file_, as_version=4) # Kernel is provided by https://github.com/django-extensions/django-extensions/ execute_preprocessor = ExecutePreprocessor(timeout=600, kernel_name='django_extensions') execute_preprocessor.preprocess(notebook, {'metadata': {'path': '.'}}) html_exporter = HTMLExporter() html_exporter.template_file = 'basic' (body, dummy_resources) = html_exporter.from_notebook_node(notebook) with open('lowfat/reports/html/{}.html'.format(notebook_filename), 'wt') as file_: file_.write(body)
def _runTest(self): kernel = 'python%d' % sys.version_info[0] cur_dir = os.path.dirname(self.nbfile) with open(self.nbfile) as f: nb = nbformat.read(f, as_version=4) if self.cov: covdict = {'cell_type': 'code', 'execution_count': 1, 'metadata': {'collapsed': True}, 'outputs': [], 'nbsphinx': 'hidden', 'source': 'import coverage\n' 'coverage.process_startup()\n' 'import sys\n' 'sys.path.append("{0}")\n'.format(cur_dir) } nb['cells'].insert(0, nbformat.from_dict(covdict)) exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600) try: run_dir = os.getenv('WRADLIB_BUILD_DIR', cur_dir) exproc.preprocess(nb, {'metadata': {'path': run_dir}}) except CellExecutionError as e: raise e if self.cov: nb['cells'].pop(0) with io.open(self.nbfile, 'wt') as f: nbformat.write(nb, f) self.assertTrue(True)
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None): ''' Execute notebook in `src` and write the output to `dst` Parameters ---------- src, dst: str path to notebook allow_errors: bool timeout: int kernel_name: str defualts to value set in notebook metadata Returns ------- dst: str ''' with io.open(src, encoding='utf-8') as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=False, timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': SOURCE_DIR}}) with io.open(dst, 'wt', encoding='utf-8') as f: nbformat.write(nb, f) return dst
def main(): arguments = docopt(__doc__, version='nbgen 2.0') cmd = subprocess.run([arguments["<path>"]] + arguments["<arguments>"], stdout=subprocess.PIPE) cmd.check_returncode() cells = json.loads(cmd.stdout.decode("utf-8")) nb_dict = { "metadata": {}, "nbformat": 4, "nbformat_minor": 0, "cells": cells, } notebook = nbformat.from_dict(nb_dict) ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(notebook, {'metadata': {}}) if arguments["nb"]: nbformat.write(notebook, "{}.ipynb".format(arguments["<name>"])) elif arguments["slides"]: config = Config() reveal_cdn = "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0/" config.SlidesExporter.reveal_url_prefix = (arguments["--reveal"] or reveal_cdn) slides, __ = export_slides(nb=notebook, config=config) with open("{}.html".format(arguments["<name>"]), "w") as html_file: html_file.write(slides)
def test_notebook(notebook): nb_name = os.path.split(notebook)[-1] if nb_name in SLOW_NOTEBOOKS: pytest.skip('Notebook is too slow to test') nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(allow_errors=False, timeout=240, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def run_ipynb(fs): """ """ with open(fs) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor() ep.preprocess(nb, {'metadata': {'path': os.path.dirname(fs)}}) with open(fs, 'wt') as f: nbformat.write(nb, fs)
def write_notebook(cells, outputfile, execute=True, kernel='python3'): kernelspec = get_kernelspec(kernel) notebook = new_notebook(cells=cells, metadata={'language': 'python', 'kernelspec': kernelspec}) if execute: ep = ExecutePreprocessor(timeout=600, kernelname='python3') ep.preprocess(notebook, {'metadata': {'path': os.path.dirname(outputfile)}}) nbformat.write(notebook, outputfile)
def _test_tutorial_nb(tutorial): """Run tutorial jupyter notebook to catch any execution error. Parameters ---------- tutorial : str tutorial name in folder/tutorial format """ tutorial_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'docs', '_build', 'html', 'tutorials') tutorial_path = os.path.join(*([tutorial_dir] + tutorial.split('/'))) # see env variable docs in the doc string of the file kernel = os.getenv('MXNET_TUTORIAL_TEST_KERNEL', None) no_cache = os.getenv('MXNET_TUTORIAL_TEST_NO_CACHE', False) working_dir = os.path.join(*([temp_dir] + tutorial.split('/'))) if no_cache == '1': print("Cleaning and setting up temp directory '{}'".format(working_dir)) shutil.rmtree(temp_dir, ignore_errors=True) errors = [] notebook = None if not os.path.isdir(working_dir): os.makedirs(working_dir) try: notebook = nbformat.read(tutorial_path + '.ipynb', as_version=IPYTHON_VERSION) # Adding a small delay to allow time for sockets to be freed # stop-gap measure to battle the 1000ms linger of socket hard coded # in the kernel API code time.sleep(1.1) if kernel is not None: eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel) else: eprocessor = ExecutePreprocessor(timeout=TIME_OUT) nb, _ = eprocessor.preprocess(notebook, {'metadata': {'path': working_dir}}) except Exception as err: err_msg = str(err) errors.append(err_msg) finally: if notebook is not None: output_file = os.path.join(working_dir, "output.txt") nbformat.write(notebook, output_file) output_nb = open(output_file, mode='r') for line in output_nb: if "Warning:" in line: errors.append("Warning:\n"+line) if len(errors) > 0: print('\n'.join(errors)) return False return True
def test_notebooks(tmpdir, filename): """ Runs a given notebook in a tmpdir """ import pylada from os.path import join, dirname from nbformat import read from nbconvert.preprocessors import ExecutePreprocessor from sys import version_info directory = join(dirname(pylada.__file__), 'notebooks') with open(join(directory, filename + ".ipynb")) as notebook_file: notebook = read(notebook_file, as_version=4) preprocessor = ExecutePreprocessor(kernel_name='python%i' % version_info.major) preprocessor.preprocess(notebook, {'metadata': {'path': str(tmpdir)}})
def run_notebook(notebook): # See http://nbconvert.readthedocs.io/en/latest/execute_api.html # TODO: Specify 'django_extensions' kernel and make it work on the server. # The kernel can be set as follows: # ep = ExecutePreprocessor(timeout=120, kernel_name='django_extensions') # This works locally, but on server, I wasn't able to create the kernel # (list available kernels by `jupyter kernelspec list`). # Default kernel currently works, given the `path` (directory from where to # execute the notebook) is set to //backend. It may fail if some Django # features are used in the notebook, but I haven't explored this. ep = ExecutePreprocessor(timeout=120) path = os.path.join(settings.REPO_DIR, 'backend') ep.preprocess(notebook, {'metadata': {'path': path}})
def translate(self): visitor = NBTranslator(self.document, self.app, self.docpath) self.document.walkabout(visitor) nb = _finilize_markdown_cells(visitor.nb) if self.app.config.nbexport_execute: ep = ExecutePreprocessor(allow_errors=True) try: ep.preprocess(nb, {'metadata': {}}) except CellExecutionError as e: self.app.warn(str(e)) self.output = nbformat.writes(nb)
def test_func(self): passing = True print("\n--------------- Testing {0} ---------------".format(nbname)) print(" {0}".format(nbpath)) if nbname in py2Ignore and sys.version_info[0] == 2: print(" Skipping {}".format(nbname)) return ep = ClearOutputPreprocessor() with open(nbpath) as f: nb = nbformat.read(f, as_version=4) ep.preprocess(nb, {}) ex = ExecutePreprocessor( timeout=600, kernel_name='python{}'.format(sys.version_info[0]), allow_errors=True ) out = ex.preprocess(nb, {}) for cell in out[0]['cells']: if 'outputs' in cell.keys(): for output in cell['outputs']: if output['output_type'] == 'error': #in output.keys(): passing = False err_msg = [] for o in output['traceback']: err_msg += ["{}".format(o)] err_msg = "\n".join(err_msg) msg = """ \n <<<<< {} FAILED >>>>> \n {} in cell [{}] \n-----------\n{}\n-----------\n ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format( nbname, output['ename'], cell['execution_count'], cell['source'], err_msg ) assert passing, msg print("\n ..... {0} Passed ..... \n".format(nbname))
def notebook_tester(fname): raw_nb = Exporter().from_filename(fname) raw_nb[0].metadata.setdefault('kernelspec', {})['name'] = 'python' preproc = ExecutePreprocessor(timeout=-1) try: exec_nb = preproc.preprocess(*raw_nb) except Exception as e: return '[Failed]\n{}'.format(e) out_nb = HTMLExporter().from_notebook_node(*exec_nb) fout = fname.replace('.ipynb', '.html') with open(fout, 'w') as f: f.write(out_nb[0]) return '[Passed]'
def execute_notebook(notebook_path: str): with open(notebook_path, encoding='utf-8') as notebook: contents = nbformat.read(notebook, as_version=4) execution_processor = ExecutePreprocessor(timeout=60, kernel_name="python3") try: # Actually execute the notebook in the current working directory. execution_processor.preprocess(contents, {'metadata': {'path': os.getcwd()}}) return True except CellExecutionError: # This is a big chunk of JSON, but the stack trace makes it reasonably # clear which cell the error occurred in, so fixing it by actually # running the notebook will probably be easier. print(contents) return False
def execute_notebook(npth, dpth, timeout=1200, kernel='python3'): """ Execute the notebook at `npth` using `dpth` as the execution directory. The execution timeout and kernel are `timeout` and `kernel` respectively. """ ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) nb = nbformat.read(npth, as_version=4) t0 = timer() ep.preprocess(nb, {'metadata': {'path': dpth}}) t1 = timer() with open(npth, 'wt') as f: nbformat.write(nb, f) return t1 - t0
def test_tutorial_notebook(): pytest.importorskip('nbformat') pytest.importorskip('nbconvert') pytest.importorskip('matplotlib') import nbformat from nbconvert.preprocessors import ExecutePreprocessor rootdir = os.path.join(aospy.__path__[0], 'examples') with open(os.path.join(rootdir, 'tutorial.ipynb')) as nb_file: notebook = nbformat.read(nb_file, as_version=nbformat.NO_CONVERT) kernel_name = 'python' + str(sys.version[0]) ep = ExecutePreprocessor(kernel_name=kernel_name) with warnings.catch_warnings(record=True): ep.preprocess(notebook, {})
def run(self): f = { 'docdir': setup.confdir, 'builddir': setup.app.builder.outdir, 'nbname': self.arguments[0], } f['nbpath'] = "{docdir}/../examples/{nbname}.ipynb".format(**f) f['destdir'] = "{builddir}/examples/{nbname}".format(**f) if not os.path.exists(f['destdir']): os.makedirs(f['destdir']) f['uneval'] = "{destdir}/{nbname}.ipynb".format(**f) f['eval'] = "{destdir}/{nbname}.eval.ipynb".format(**f) f['py'] = "{destdir}/{nbname}.py".format(**f) # 1. Uneval notebook shutil.copyfile(f['nbpath'], f['uneval']) with open(f['nbpath']) as nb_f: nb = nbformat.read(nb_f, as_version=4) # 2. Python export_python(nb, f['py']) # 3. HTML (execute first) executer = ExecutePreprocessor(timeout=240) executer.preprocess(nb, {}) html = export_html(nb, f) # 4. Eval'd notebook with open(f['eval'], 'w') as eval_f: nbformat.write(nb, eval_f) # Create link to notebook and script files link_rst = "({uneval}; {eval}; {py})".format( uneval=formatted_link(f['uneval']), eval=formatted_link(f['eval']), py=formatted_link(f['py']), ) rst_file = self.state_machine.document.attributes['source'] self.state_machine.insert_input([link_rst], rst_file) # create notebook node nb_node = notebook_node('', html, format='html', source='nb_path') nb_node.source, nb_node.line = (self.state_machine .get_source_and_line(self.lineno)) # add dependency self.state.document.settings.record_dependencies.add(f['nbpath']) return [nb_node]
def execute_nb(nb): kernel_name = nb.metadata.get('kernelspec', {}).get('name', 'python') km, kc = start_new_kernel( kernel_name=kernel_name, stderr=open(os.devnull, 'w'), cwd=None) kc.allow_stdin = False try: ep = ExecutePreprocessor(timeout=-1, kernel_name=kernel_name) nb, resources = ep.preprocess(nb, {'metadata': {'path': './'}}) finally: kc.stop_channels() km.shutdown_kernel(now=True) return nb
def execute_notebook(notebook_filename, notebook_filename_out, params_dict, run_path="", timeout=6000000): notebook_fp = os.path.join(run_path, notebook_filename) nb = read_in_notebook(notebook_fp) new_nb = set_parameters(nb, params_dict) ep = ExecutePreprocessor(timeout=timeout, kernel_name='python3') try: ep.preprocess(new_nb, {'metadata': {'path': run_path}}) except: msg = 'Error executing the notebook "{0}".\n\n'.format(notebook_filename) msg = '{0}See notebook "{1}" for the traceback.'.format(msg, notebook_filename_out) print(msg) raise finally: with open(notebook_filename_out, mode='wt') as f: nbformat.write(new_nb, f)
def run_notebook(notebook_name, nb_suffix='-out', out_path='.', timeout=3600, **execute_kwargs): """Runs a notebook and saves the output in the same notebook. Arguments: notebook_name (string): name of the notebook to be executed. nb_suffix (string): suffix to append to the file name of the executed notebook. timeout (int): timeout in seconds after which the execution is aborted. execute_kwargs (dict): additional arguments passed to `ExecutePreprocessor`. out_path (string): folder where to save the output notebook. """ timestamp_cell = "**Executed:** %s\n\n**Duration:** %d seconds." if str(notebook_name).endswith('.ipynb'): notebook_name = str(notebook_name)[:-len('.ipynb')] nb_name_input = notebook_name + '.ipynb' nb_name_output = notebook_name + '%s.ipynb' % nb_suffix nb_name_output = os.path.join(out_path, nb_name_output) print('- Executing: ', nb_name_input) execute_kwargs_ = dict(kernel_name = 'python%d' % sys.version_info[0]) if execute_kwargs is not None: execute_kwargs_.update(execute_kwargs) ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs_) nb = nbformat.read(nb_name_input, as_version=4) start_time = time.time() try: # Execute the notebook ep.preprocess(nb, {'metadata': {'path': './'}}) except: # Execution failed, print a message then raise. msg = 'Error executing the notebook "%s".\n\n' % notebook_name msg += 'See notebook "%s" for the traceback.' % nb_name_output print(msg) raise else: # On successful execution, add timestamping cell duration = time.time() - start_time timestamp_cell = timestamp_cell % (time.ctime(start_time), duration) nb['cells'].insert(0, nbformat.v4.new_markdown_cell(timestamp_cell)) finally: # Save the notebook even when it raises an error nbformat.write(nb, nb_name_output) print('* Output: ', nb_name_output)
def test_tutorial_nb(file_path, workingdir, kernel=None): """Run tutorial jupyter notebook to catch any execution error. Parameters ---------- file_path : str path of tutorial .ipynb file workingdir: str path of the directory to run the tutorial in kernel: str Default None name of the kernel to use, if none, will use first kernel in the list """ tutorial_name = os.path.basename(file_path) sys.stdout.write('Testing {}...'.format(file_path)) sys.stdout.flush() tick = time.time() notebook = nbformat.read(file_path + '.ipynb', as_version=4) if kernel: eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel) else: eprocessor = ExecutePreprocessor(timeout=TIME_OUT) success = True try: eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}}) except Exception as err: err_msg = str(err) fail_dict[tutorial_name] = err_msg success = False finally: output_file = os.path.join(workingdir, "output.txt") output_nb = open(output_file, mode='w') nbformat.write(notebook, output_nb) output_nb.close() output_nb = open(output_file, mode='r') for line in output_nb: if "Warning:" in line: success = False if tutorial_name in fail_dict: fail_dict[tutorial_name] += "\n"+line else: fail_dict[tutorial_name] = "Warning:\n"+line sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick )) sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed')) sys.stdout.flush()
def nbexec(nb_base, nb): base_name = nb_base.split('_')[1].split('.')[0].upper() afp_name = nb.split('_')[1].split('.')[0].upper() # Lee nb_base origen with io.open(nb_base, 'rt', encoding="utf8") as f: aux = nbformat.read(f, as_version=4) # Reemplaza afp_name u = aux['cells'][2]['source'].replace(base_name, afp_name) aux['cells'][2]['source'] = u # Opciones ep = ExecutePreprocessor(timeout=300) # Procesa aux (Revisar si preprocess modifica aux) ep.preprocess(aux, {}) # Escribe nb destino with io.open(nb, 'wt', encoding="utf8") as f: nbformat.write(aux, f) print "[INFO]--" + datetime.now().strftime('%Y-%M-%d %H:%M:%S') + "--" + "nbexec" + "--" + nb
def execute(self, write=True): """ Execute the specified notebook file, and optionally write out the executed notebook to a new file. Parameters ---------- write : bool, optional Write the executed notebook to a new file, or not. Returns ------- executed_nb_path : str, ``None`` The path to the executed notebook path, or ``None`` if ``write=False``. """ if path.exists(self._executed_nb_path) and not self.overwrite: logger.debug("Executed notebook already exists at {0}. Use " "overwrite=True or --overwrite (at cmd line) to re-run" .format(self._executed_nb_path)) return self._executed_nb_path # Execute the notebook logger.debug('Executing notebook using kwargs ' '"{}"...'.format(self._execute_kwargs)) executor = ExecutePreprocessor(**self._execute_kwargs) with open(self.nb_path) as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) try: executor.preprocess(nb, {'metadata': {'path': self.path_only}}) except CellExecutionError: # TODO: should we fail fast and raise, or record all errors? raise if write: logger.debug('Writing executed notebook to file {0}...' .format(self._executed_nb_path)) with open(self._executed_nb_path, 'w') as f: nbformat.write(nb, f) return self._executed_nb_path
def _aux_funct_notebook(self, notebook_filename): assert os.path.exists( notebook_filename), f"{notebook_filename} do not exists!" with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) try: ep = ExecutePreprocessor(timeout=60, store_widget_state=True) try: ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_PATHS}}) except CellExecutionError as exc_: raise except Exception as exc_: # error with tqdm progress bar i believe pass except CellExecutionError as exc_: raise except Exception: pass
def nbexec(nb_base, nb): base_name = nb_base.split('_')[1].split('.')[0].upper() afp_name = nb.split('_')[1].split('.')[0].upper() # Lee nb_base origen with io.open(nb_base, 'rt', encoding="utf8") as f: aux = nbformat.read(f, as_version=4) # Reemplaza afp_name u = aux['cells'][2]['source'].replace(base_name, afp_name) aux['cells'][2]['source'] = u # Opciones ep = ExecutePreprocessor(timeout=300) # Procesa aux (Revisar si preprocess modifica aux) ep.preprocess(aux, {}) # Escribe nb destino with io.open(nb, 'wt', encoding="utf8") as f: nbformat.write(aux, f) print "[INFO]--" + datetime.now().strftime( '%Y-%M-%d %H:%M:%S') + "--" + "nbexec" + "--" + nb
def _run_notebook(filename): # Create the preprocessor. execute_preprocessor = ExecutePreprocessor(timeout=TIMEOUT, kernel_name=JUPYTER_KERNEL) # Open the notebook. file_path = os.path.dirname(os.path.abspath(filename)) with open(filename) as file_: notebook = nbformat.read(file_, as_version=4) with warnings.catch_warnings(): # Silence some spurious warnings. warnings.filterwarnings('ignore', category=DeprecationWarning) # Finally, run the notebook. execute_preprocessor.preprocess(notebook, {'metadata': { 'path': file_path }})
def test_reexecute_result_notebook(): with exec_for_test('define_hello_world_pipeline') as result: assert result.success materialization_events = [ x for x in result.step_event_list if x.event_type_value == 'STEP_MATERIALIZATION' ] for materialization_event in materialization_events: result_path = get_path(materialization_event) if result_path.endswith('.ipynb'): with open(result_path) as fd: nb = nbformat.read(fd, as_version=4) ep = ExecutePreprocessor() ep.preprocess(nb, {}) with open(result_path) as fd: assert nbformat.read(fd, as_version=4) == nb
def run_notebook(filename): run_path = os.path.split(filename)[0] with open(filename) as f: nb = nbformat.read(f, as_version=4) try: ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(nb, {'metadata': {'path': run_path}}) # FIXME: use tempfile and mv to avoid interruption # better split the source code of the notebook and the compiled targets. with open(filename, 'wt') as f: nbformat.write(nb, f) except Exception as e: print('processing', filename, e)
def run(self, nb_name=None, cell_timeout=60 * 60, remove_config=True, **kwargs): """Run the notebook Paramters --------- nb_name: str or Path (name of step) name of the notebook, assumed to be in same directory as step cell_timeout: int (3600) number of seconds to allow each cell to execut remove_config: bool (True) don't keep serialized version of this class that provides info to the notebook """ # Find path and name of notebook nb_path = Path(__file__).parent.parent / "steps" / self.step_name if nb_name is None: nb_name = self.name if not nb_name.endswith(".ipynb"): nb_name = nb_name + ".ipynb" # Write out pickled self for notebook config self.args = kwargs self.interactive = False # to inform notebooks config_path = nb_path / "config.pkl" with open(config_path, "wb") as file: cloudpickle.dump(self, file) # Run notebook with open(nb_path / nb_name) as file: nb = nbformat.read(file, as_version=4) executer = ExecutePreprocessor(timeout=cell_timeout, kernel_name="python3") executer.preprocess(nb, {"metadata": {"path": str(nb_path)}}) # Reload pickled self from config with open(config_path, "rb") as file: self = cloudpickle.load(file) self.manifest.to_csv(self.step_local_staging_dir / Path("manifest.csv"), index=False) if remove_config: config_path.unlink() return
def test_radial_fourier_default(hdf5_ds_2, tmpdir_factory, lt_ctx): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'local'}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) analysis = [{ "analysisType": "RADIAL_FOURIER", "parameters": { 'shape': 'radial_fourier', 'cx': 0, 'cy': 0, 'ri': 0, 'ro': 2, 'n_bins': 2, 'max_order': 23, } }] notebook = notebook_generator(conn, dataset, analysis, save=True) notebook = io.StringIO(notebook.getvalue()) nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(timeout=600) out = ep.preprocess(nb, {"metadata": {"path": datadir}}) channels = ["dominant_0", "absolute_0_0", "absolute_0_1"] results = {} for channel in channels: data_path = os.path.join(datadir, f"radial_result_{channel}.npy") results[channel] = np.load(data_path) analysis = lt_ctx.create_radial_fourier_analysis(dataset=hdf5_ds_2, cx=0, cy=0, ri=0, ro=2, n_bins=2, max_order=23) expected = lt_ctx.run(analysis) assert np.allclose(results["dominant_0"], expected["dominant_0"].raw_data) assert np.allclose(results["absolute_0_0"], expected["absolute_0_0"].raw_data) assert np.allclose(results["absolute_0_1"], expected["absolute_0_1"].raw_data)
def test_com_default(hdf5_ds_2, tmpdir_factory, lt_ctx, local_cluster_url): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) analysis = [{ "analysisType": 'CENTER_OF_MASS', "parameters": { 'shape': 'com', 'cx': 0, 'cy': 0, 'r': 8, 'ri': 4, } }] notebook = notebook_generator(conn, dataset, analysis, save=True) notebook = io.StringIO(notebook.getvalue()) nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(timeout=30) ep.preprocess(nb, {"metadata": {"path": datadir}}) channels = ["field", "magnitude", "divergence", "curl", "x", "y"] results = {} for channel in channels: data_path = os.path.join(datadir, f"com_result_{channel}.npy") results[channel] = np.load(data_path) com_analysis = lt_ctx.create_com_analysis( dataset=hdf5_ds_2, cx=0, cy=0, mask_radius=8, mask_radius_inner=4, ) expected = lt_ctx.run(com_analysis) assert np.allclose(results["field"], expected["field"].raw_data) assert np.allclose(results["magnitude"], expected["magnitude"].raw_data) assert np.allclose(results["divergence"], expected["divergence"].raw_data) assert np.allclose(results["curl"], expected["curl"].raw_data) assert np.allclose(results["x"], expected["x"].raw_data) assert np.allclose(results["y"], expected["y"].raw_data)
def run_notebook(notebook, root_path=".", timeout=30, prerun=None): """Run a notebook in Jupyter This function will copy all of the files in ``root_path`` to a temporary directory, run the notebook and then return a ``NotebookResult`` object containing the outputs for each cell. The notebook is run in a separate process and only objects that are serializable will be returned in their entirety, otherwise the string representation will be returned instead. Parameters ---------- notebook : str The notebook to run relative to ``root_path`` root_path : str The root notebook folder (default ".") timeout : int Length of time to run the notebook in seconds (default 30) prerun : function Function to run prior to starting the notebook, takes the temporary copy of root_path as a parameter """ import nbformat from nbconvert.preprocessors import ExecutePreprocessor with tempfile.TemporaryDirectory() as td: workdir = os.path.join(td, 'work') notebook_dir = os.path.join(workdir, os.path.dirname(notebook)) shutil.copytree(root_path, workdir) if prerun is not None: prerun(workdir) fullpath = os.path.join(workdir, notebook) with open(fullpath, "r") as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(kernel_name='python3', timeout=timeout) code_cells = [c for c in nb['cells'] if c['cell_type'] == 'code'] nb['cells'].append( nbformat.from_dict({'cell_type': 'code', 'metadata': {}, 'source': _create_code(len(code_cells))} )) ep.preprocess(nb, {'metadata': {'path': notebook_dir}}) return NotebookResult(nb)
def run_notebook(self, nb, f): """Runs a loaded notebook file.""" if PYTHON_MAJOR_VERSION == 3: kernel_name = 'python3' elif PYTHON_MAJOR_VERSION == 2: kernel_name = 'python2' else: raise Exception('Only Python 2 and 3 are supported') ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) try: ep.preprocess(nb, {'metadata': {'path': '.'}}) except CellExecutionError: msg = 'Error executing the notebook "%s".\n\n' % f.name msg += 'See notebook "%s" for the traceback.' % f.name print(msg) raise finally: nbformat.write(nb, f)
def run_notebook(notebook_path: str) -> Tuple[Any, List[Any]]: full_path = os.path.join(ROOT_DIR, notebook_path) nb_name, _ = os.path.splitext(os.path.basename(full_path)) with open(full_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': os.path.dirname(full_path)}}) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def test_notebook(path): if path in MARS: if not os.path.exists(os.path.expanduser("~/.ecmwfapirc")): pytest.skip("No ~/.ecmwfapirc") if path in CDS: if not os.path.exists(os.path.expanduser("~/.cdsapirc")): pytest.skip("No ~/.cdsapirc") if path in TENSORFLOW: if sys.version_info > (3, 8): pytest.skip("Tensorflow not yet ready on 3.9") with open(os.path.join(EXAMPLES, path)) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=60 * 60, kernel_name="python3") proc.preprocess(nb, {"metadata": {"path": EXAMPLES}})
def test_reexecute_result_notebook(): with exec_for_test( "hello_world_pipeline", {"loggers": {"console": {"config": {"log_level": "ERROR"}}}} ) as result: assert result.success materialization_events = [ x for x in result.step_event_list if x.event_type_value == "STEP_MATERIALIZATION" ] for materialization_event in materialization_events: result_path = get_path(materialization_event) if result_path.endswith(".ipynb"): with open(result_path) as fd: nb = nbformat.read(fd, as_version=4) ep = ExecutePreprocessor() ep.preprocess(nb, {}) with open(result_path) as fd: assert nbformat.read(fd, as_version=4) == nb
def run_notebook(filename): execution_failed = False with open(filename) as f: nb = nbformat.read(f, as_version=4) try: ep = ExecutePreprocessor(timeout=None, kernel_name='python3') ep.preprocess(nb, {'metadata': {'path': './'}}) except Exception as e: print( "[" + datetime.now().time().strftime('%H:%M') + "] " + "Error in file '", filename, "': ", str(e).split('\n')[-2]) execution_failed = True if not execution_failed: with open(filename, 'w', encoding='utf-8') as f: nbformat.write(nb, f) return 1 return 0
def test_run_data_assistants_notebook(tmp_path): """ What does this test and why? One of the resources we have for DataAssistants is a Jupyter notebook that explains/shows the components in code. This test ensures the codepaths and examples described in the Notebook actually run and pass, nbconvert's `preprocess` function. """ base_dir: str = file_relative_path( __file__, "../../../test_fixtures/rule_based_profiler/example_notebooks") notebook_path: str = os.path.join( base_dir, "DataAssistants_Instantiation_And_Running.ipynb") # temporary output notebook for traceback and debugging output_notebook_path: str = os.path.join( tmp_path, "DataAssistants_Instantiation_and_running_executed.ipynb") with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=60, kernel_name="python3") try: ep.preprocess(nb, {"metadata": {"path": base_dir}}) except CellExecutionError: msg = 'Error executing the notebook "%s".\n\n' % notebook_path msg += 'See notebook "%s" for the traceback.' % output_notebook_path print(msg) raise finally: with open(output_notebook_path, mode="w", encoding="utf-8") as f: nbformat.write(nb, f) # clean up Expectations directory after running test shutil.rmtree(os.path.join(base_dir, "great_expectations/expectations/tmp")) os.remove( os.path.join(base_dir, "great_expectations/expectations/.ge_store_backend_id")) os.remove( os.path.join(base_dir, "great_expectations/expectations/taxi_data_suite.json"))
def matplotlib_notebook(tmpdir, request): nb = nbformat.v4.new_notebook() nb.cells.append(nbformat.v4.new_code_cell('\n'.join( ['import matplotlib.pyplot as plt', '%matplotlib inline']))) for _ in range(request.param): nb.cells.append(nbformat.v4.new_code_cell('\n'.join( ['plt.figure(dpi=100)', 'plt.plot(range(100))', 'plt.show()']))) ep = ExecutePreprocessor() ep.preprocess(nb, {'metadata': {'path': tmpdir}}) nb['metadata'].update({'path': f'{tmpdir}'}) return nb
def __init__(self, notebook_filename): """ """ self.outfilename = notebook_filename # Generate an empty notebook self.nb = nbf.v4.new_notebook() # Get all the import from NotebookConfig self._import() # Set up some display options for pandas to extend # the limit of the row and columns that are displayed self._setup_display() # Setup matplotlib magic and size of figures self._setup_matplotlib() # Load the dataset self._load_dataset() # Processor to run the notebook self.processor = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=self.allow_errors)
def execute_notebook(source): """ :param source: Jupyter Notebook :return: Result of the notebook invocation """ logger.debug("Executing notebook") in_memory_source = StringIO(source) nb = nbformat.read(in_memory_source, as_version=4) logger.debug("Launching kernels") ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=True) ep.preprocess(nb, {'metadata': {'path': '/tmp/'}}) ex = StringIO() nbformat.write(nb, ex) logger.debug("Returning results") return ex.getvalue()
def test_sum_default(hdf5_ds_2, tmpdir_factory): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'local'}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) analysis = [{"analysisType": "SUM_FRAMES", "parameters": {"roi": {}}}] notebook = notebook_generator(conn, dataset, analysis, save=True) notebook = io.StringIO(notebook.getvalue()) nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(timeout=600) out = ep.preprocess(nb, {"metadata": {"path": datadir}}) data_path = os.path.join(datadir, 'sum_result.npy') result = np.load(data_path) with hdf5_ds_2.get_reader().get_h5ds() as h5ds: data = h5ds[:] expected = data.sum(axis=(0, 1)) assert np.allclose(expected, result)
def test_notebook(notebook): fullfile = os.path.abspath(notebook) _, filename = os.path.split(fullfile) filename, _ = os.path.splitext(filename) if filename in KNOWN_FAILURES: pytest.skip('{0} is known to fail'.format(filename)) if filename in RPY2_NOTEBOOKS and not HAS_RPY2: pytest.skip('{0} since rpy2 is not installed'.format(filename)) if filename in JOBLIB_NOTEBOOKS and not JOBLIB_NOTEBOOKS: pytest.skip('{0} since joblib is not installed'.format(filename)) with io.open(fullfile, encoding='utf-8') as fp: nb = nbformat.read(fp, as_version=4) ep = ExecutePreprocessor(allow_errors=False, timeout=20, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def run_notebook(notebook_path): assert os.path.isfile(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) logging.info("Running: {}".format(notebook_path)) ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(nb, {'metadata': {'path': 'notebooks/'}}) exec_name = os.path.join( os.path.dirname(notebook_path), '_'.join(['executed', os.path.basename(notebook_path)]) ) logging.info("Saving results to: {}".format(exec_name)) with open(exec_name, 'w', encoding='utf-8') as f: nbformat.write(nb, f)
def run_notebook(self, nb_name): """Runs a notebook and write out the output notebook Args: nb_name (string): notebook filename """ out_notebook_name = 'output_notebooks/%s_result_nb.ipynb' % nb_name nb = nbformat.read('../data_cleaning/%s' % nb_name, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name='python3') try: ep.preprocess(nb, {'metadata': {'path': '../data_cleaning/'}}) except CellExecutionError: msg = 'Error executing the notebook' msg += 'See notebook "%s" for the traceback.' % out_notebook_name self.logger.info(msg) raise finally: with open(out_notebook_name, mode='wt') as f: nbformat.write(nb, f)
def setUp(self): self.path_test_nb = path.join(PATH_FIXTURES, self.name + ".ipynb") self.path_test_html = path.join(PATH_FIXTURES, self.name + ".html") self.path_test_js = path.join(PATH_JS_TESTS, self.name + ".js") self.kernel_name = kernelspec.KERNEL_NAME with open(self.path_test_nb, "r") as f: self.nb = nbformat.read(f, as_version=4) self.ep = ExecutePreprocessor(timeout=600, kernel_name=self.kernel_name) self.html_exporter = HTMLExporter() self.ep.preprocess(self.nb, {"metadata": {"path": "."}}) (self.body, _) = self.html_exporter.from_notebook_node(self.nb) with open(self.path_test_html, "w") as f: f.write(self.body)
def retrieve_errors(nb_path): """ Find all errors that occur when running a nb """ with nb_path.open() as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=True) out, _ = ep.preprocess(nb, {'metadata': {'path': 'nbs/'}}) errors = [] for cell in out.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output.evalue) return errors
def test(self): PYMOO_DIR = get_pymoo() DOC_DIR = os.path.join(PYMOO_DIR, "doc", "source") ipynb = [] # collect all the jupyter ipynb in the documentation for root, directories, filenames in os.walk(DOC_DIR): for filename in filenames: if filename.endswith(".ipynb") and "checkpoint" not in filename and not any([filename in s for s in SKIP]): ipynb.append(os.path.join(root, filename)) i = 0 if STARTING_AT is not None: for j in range(len(ipynb)): if STARTING_AT not in ipynb[j]: i += 1 else: break ep = ExecutePreprocessor(timeout=10000, kernel_name='python3') for i in range(i, len(ipynb)): fname = ipynb[i] print(fname.split("/")[-1]) import warnings warnings.filterwarnings("ignore") try: nb = nbformat.read(fname, nbformat.NO_CONVERT) ep.preprocess(nb, {'metadata': {'path': PYMOO_DIR}}) except CellExecutionError: msg = 'Error executing the fname "%s".\n\n' % fname print(msg) raise finally: if OVERWRITE: with open(fname, mode='wt') as f: nbformat.write(nb, f)
def run_ipynb(file_path): """Execute Jupyter notebook programmatically. After all cells in the existing notebook have beer run, the function overwrites it. As a fail-safe, the existing notebook in its pre-run state is saved to 'file_path.parent/.old/'. Parameters ---------- file_path : str, pathlib.Path Path to notebook to be run. References ---------- 1. https://nbconvert.readthedocs.io/en/latest/execute_api.html 2. Steven F. Lott, "Modern Python Cookbook", Chapter 9, "Replacing a file while preserving the previous version", pp. 437-440, 2016. """ file_path = Path(file_path) file_path_temp = file_path.with_suffix('.ipynb.tmp') file_path.parent.joinpath('.old').mkdir(parents=False, exist_ok=True) file_path_old = (file_path .parent .joinpath('.old', file_path.name) .with_suffix('.ipynb.old')) with open(file_path, 'r', encoding='utf-8') as nb: nbook = nbformat.read(nb, as_version=4) ep = ExecutePreprocessor(kernel_name='python3') ep.preprocess(nbook, {'metadata': {'path': file_path.parent}}) with open(file_path_temp, 'w', encoding='utf-8') as nb: nbformat.write(nbook, nb) try: file_path_old.unlink() except FileNotFoundError: pass file_path.rename(file_path_old) file_path_temp.rename(file_path)
def test_example_notebooks(): """ Tests that all the example ipython notebooks of format surfinBH/examples/example_*.ipynb are working. Since we expect these to be used by our users, it would be emabarassing if our own examples failed. """ # List of all available fits fit_names = surfinBH.fits_collection.keys() for name in fit_names: short_name = name.split('surfinBH')[-1] notebook_filename = 'examples/example_%s.ipynb'%short_name print('testing %s'%notebook_filename) with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=None, kernel_name=python_version) ep.preprocess(nb, {'metadata': {'path': '.'}})
def execute_notebook(notebook_path: str): with open(notebook_path, encoding='utf-8') as notebook: contents = nbformat.read(notebook, as_version=4) execution_processor = ExecutePreprocessor(timeout=60, kernel_name="python3") try: # Actually execute the notebook in the current working directory. execution_processor.preprocess(contents, {'metadata': { 'path': os.getcwd() }}) return True except CellExecutionError: # This is a big chunk of JSON, but the stack trace makes it reasonably # clear which cell the error occurred in, so fixing it by actually # running the notebook will probably be easier. print(contents) return False
def setUpClass(cls): print('Running modified notebook...') notebook_path = os.path.join( os.path.dirname(os.path.dirname(__file__)), 'demo.ipynb') kernel_name = "python{0}".format(sys.version_info.major) with open(notebook_path) as notebook_file: cls.notebook = nbformat.read(notebook_file, nbformat.current_nbformat) cls.notebook = _modify_grid_search_cell(cls.notebook) ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name, allow_errors=True) ep.preprocess( cls.notebook, {'metadata': { 'path': os.path.dirname(notebook_path) }})
def execute_notebook(src='estimate.src.ipynb', dest='estimate.ipynb'): """Executes the analysis notebook and writes out a copy with all of the resulting tables and plots. Parameters ---------- src: str, optional Source notebook to execute dest: str, optional Output notebook """ with open(src) as fp: nb = nbformat.read(fp, 4) exp = ExecutePreprocessor(timeout=300) updated_nb, _ = exp.preprocess(nb, {}) with open(dest, 'w') as fp: nbformat.write(updated_nb, fp)