def test_tutorial_nb(file_path): """Run tutorial jupyter notebook to catch any execution error. Parameters ---------- file_path : str path of tutorial markdown file """ tutorial_name = os.path.basename(file_path) notebook = nbformat.read(file_path + '.ipynb', as_version=4) eprocessor = ExecutePreprocessor(timeout=1800) try: eprocessor.preprocess(notebook, {'metadata': {}}) except Exception as err: err_msg = str(err) fail_dict[tutorial_name] = err_msg finally: output_nb = open("output.txt", mode='w') nbformat.write(notebook, output_nb) output_nb.close() output_nb = open("output.txt", mode='r') for line in output_nb: if "Warning:" in line: fail_dict[tutorial_name] = "%s has warning." % (tutorial_name) return
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=''): """ Execute notebook in `src` and write the output to `dst` Parameters ---------- src, dst: str path to notebook allow_errors: bool timeout: int kernel_name: str defualts to value set in notebook metadata Returns ------- dst: str """ import nbformat from nbconvert.preprocessors import ExecutePreprocessor with io.open(src, encoding='utf-8') as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, resources={}) with io.open(dst, 'wt', encoding='utf-8') as f: nbformat.write(nb, f) return dst
def _notebook_run(path): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ kernel_name = "python%d" % sys.version_info[0] this_file_directory = os.path.dirname(__file__) errors = [] with tempfile.NamedTemporaryFile(suffix=".ipynb", mode="wt") as fout: with open(path) as f: nb = nbformat.read(f, as_version=4) nb.metadata.get("kernelspec", {})["name"] = kernel_name ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10) try: ep.preprocess(nb, {"metadata": {"path": this_file_directory}}) except CellExecutionError as e: if "SKIP" in e.traceback: print(str(e.traceback).split("\n")[-2]) else: raise e except TimeoutError as e: print(e) finally: nbformat.write(nb, fout) # nb = nbformat.read(fout, nbformat.current_nbformat) # errors = errors.extend( # [output for cell in nb.cells if "outputs" in cell # for output in cell["outputs"] if output.output_type == "error"]) return nb, errors
def run_notebook(notebook_name, nb_kwargs=None, insert_pos=5, timeout=120, execute_kwargs=None): """Runs a notebook and displays the output in the master notebook. Executes a notebook, optionally passing "arguments" in a way roughly similar to passing arguments to a function. Notebook arguments are passed in a dictionary (`nb_kwargs`) which is converted to a string containing python code, then inserted in the notebook as a code cell. The code contains only assignments of variables which can be used to control the execution of a suitably written notebook. When calling a notebook, you need to know which arguments (variables) to pass. Differently from functions, no check on the input arguments is performed. Arguments: notebook_name (string): name of the notebook to be executed. nb_kwargs (dict or None): If not None, this dict is converted to a string of python assignments with keys representing variables names and values variables content. This string is inserted as code-cell in the notebook to be executed. insert_pos (int): position of insertion of the code-cell containing the input arguments. Default is 5 (i.e. sixth cell). With this default, the input notebook can define, in the first cell, default values of input arguments (used when the notebook is executed with no arguments or through the Notebook GUI). timeout (int): timeout in seconds after which the execution is aborted. execute_kwargs (dict): additional arguments passed to `ExecutePreprocessor`. """ if nb_kwargs is not None: header = '# Cell inserted during automated execution.' code = dict_to_code(nb_kwargs) code_cell = '\n'.join((header, code)) if execute_kwargs is None: execute_kwargs = {} ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs) nb = nbformat.read(notebook_name, as_version=4) if len(nb_kwargs) > 0: nb['cells'].insert(insert_pos, nbformat.v4.new_code_cell(code_cell)) try: # Execute the notebook ep.preprocess(nb, {'metadata': {'path': './'}}) #***************************************** # Title: analysis.ipynb # Author: Benjamin RK # Date: April 2013 # Availability: https://gist.github.com/minrk/5491090 ip = get_ipython() for cell in nb.cells: if cell.cell_type != 'code': continue ip.run_cell(cell.source) #***************************************** except: # Execution failed, print a message then raise. msg = 'Error executing the notebook "%s".\n\n' % notebook_name print(msg) raise
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None): ''' Execute notebook in `src` and write the output to `dst` Parameters ---------- src, dst: str path to notebook allow_errors: bool timeout: int kernel_name: str defualts to value set in notebook metadata Returns ------- dst: str ''' with io.open(src, encoding='utf-8') as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=False, timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': SOURCE_DIR}}) with io.open(dst, 'wt', encoding='utf-8') as f: nbformat.write(nb, f) return dst
def execute(self): print("Cleaning lowfat/reports/html ...") old_reports = os.listdir("lowfat/reports/html") for old_report in old_reports: print("- Removing lowfat/reports/html/{}".format(old_report)) os.remove("lowfat/reports/html/{}".format(old_report)) print("Cleaning of lowfat/reports/html is complete.") notebook_filenames = os.listdir("lowfat/reports") for notebook_filename in notebook_filenames: if not notebook_filename.endswith(".ipynb"): continue print("Processing lowfat/reports/{}".format(notebook_filename)) # Based on Executing notebooks, nbconvert Documentation by Jupyter Development Team. # https://nbconvert.readthedocs.io/en/latest/execute_api.html with open("lowfat/reports/{}".format(notebook_filename)) as file_: notebook = nbformat.read(file_, as_version=4) # Kernel is provided by https://github.com/django-extensions/django-extensions/ execute_preprocessor = ExecutePreprocessor(timeout=600, kernel_name='django_extensions') execute_preprocessor.preprocess(notebook, {'metadata': {'path': '.'}}) html_exporter = HTMLExporter() html_exporter.template_file = 'basic' (body, dummy_resources) = html_exporter.from_notebook_node(notebook) with open('lowfat/reports/html/{}.html'.format(notebook_filename), 'wt') as file_: file_.write(body)
def nb_to_html(root, template='basic', version=4, timeout=600, kernel='python3'): ''' This functions executes a Jupyter notebook and creates the related HTML file. Args: root (str): name of the file without the .ipynb extension template (str): name of the template (to be in the current folder as template.tpl) version (int): version of the notebook timeout (float): maximum time spent per cell kernel (str) Returns: None The function executes root.ipynb into root_exe.ipynb and creates the file root.html. ''' with open(root + '.ipynb') as f: nb = nbformat.read(f, as_version=version) ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) ep.preprocess(nb, {'metadata': {'path': '.'}}) with open(root + '_exe.ipynb', 'wt') as f: nbformat.write(nb, f) html_exporter = HTMLExporter() html_exporter.template_file = template with open(root + '_exe.ipynb', mode='r') as f: notebook = nbformat.reads(''.join(f.readlines()), as_version=version) (body, _) = html_exporter.from_notebook_node(notebook) codecs.open(root + '.html', 'w', encoding='utf-8').write(body)
def main(): arguments = docopt(__doc__, version='nbgen 2.0') cmd = subprocess.run([arguments["<path>"]] + arguments["<arguments>"], stdout=subprocess.PIPE) cmd.check_returncode() cells = json.loads(cmd.stdout.decode("utf-8")) nb_dict = { "metadata": {}, "nbformat": 4, "nbformat_minor": 0, "cells": cells, } notebook = nbformat.from_dict(nb_dict) ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(notebook, {'metadata': {}}) if arguments["nb"]: nbformat.write(notebook, "{}.ipynb".format(arguments["<name>"])) elif arguments["slides"]: config = Config() reveal_cdn = "https://cdnjs.cloudflare.com/ajax/libs/reveal.js/3.3.0/" config.SlidesExporter.reveal_url_prefix = (arguments["--reveal"] or reveal_cdn) slides, __ = export_slides(nb=notebook, config=config) with open("{}.html".format(arguments["<name>"]), "w") as html_file: html_file.write(slides)
def _runTest(self): kernel = 'python%d' % sys.version_info[0] cur_dir = os.path.dirname(self.nbfile) with open(self.nbfile) as f: nb = nbformat.read(f, as_version=4) if self.cov: covdict = {'cell_type': 'code', 'execution_count': 1, 'metadata': {'collapsed': True}, 'outputs': [], 'nbsphinx': 'hidden', 'source': 'import coverage\n' 'coverage.process_startup()\n' 'import sys\n' 'sys.path.append("{0}")\n'.format(cur_dir) } nb['cells'].insert(0, nbformat.from_dict(covdict)) exproc = ExecutePreprocessor(kernel_name=kernel, timeout=600) try: run_dir = os.getenv('WRADLIB_BUILD_DIR', cur_dir) exproc.preprocess(nb, {'metadata': {'path': run_dir}}) except CellExecutionError as e: raise e if self.cov: nb['cells'].pop(0) with io.open(self.nbfile, 'wt') as f: nbformat.write(nb, f) self.assertTrue(True)
def _notebook_run(path): """Execute a notebook via nbconvert and collect output. :returns (parsed nb object, execution errors) """ kernel_name = 'python%d' % sys.version_info[0] this_file_directory = os.path.dirname(__file__) errors = [] with tempfile.NamedTemporaryFile(suffix=".ipynb", mode='wt') as fout: with open(path) as f: nb = nbformat.read(f, as_version=4) nb.metadata.get('kernelspec', {})['name'] = kernel_name ep = ExecutePreprocessor(kernel_name=kernel_name, timeout=10) try: ep.preprocess(nb, {'metadata': {'path': this_file_directory}}) except CellExecutionError as e: if "SKIP" in e.traceback: print(str(e.traceback).split("\n")[-2]) else: raise e except RuntimeError as e: print(e) finally: nbformat.write(nb, fout) return nb, errors
def execute_nb(src, dst, allow_errors=False, timeout=1000, kernel_name=None): """ Execute notebook in `src` and write the output to `dst` Parameters ---------- src, dst: str path to notebook allow_errors: bool timeout: int kernel_name: str defualts to value set in notebook metadata Returns ------- dst: str """ with io.open(src, encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=allow_errors, timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, {"metadta": {"path": "notebooks/"}}) with io.open(dst, "wt", encoding="utf-8") as f: nbformat.write(nb, f) return dst
def test_notebook(notebook): nb_name = os.path.split(notebook)[-1] if nb_name in SLOW_NOTEBOOKS: pytest.skip('Notebook is too slow to test') nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(allow_errors=False, timeout=240, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def run_ipynb(fs): """ """ with open(fs) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor() ep.preprocess(nb, {'metadata': {'path': os.path.dirname(fs)}}) with open(fs, 'wt') as f: nbformat.write(nb, fs)
def write_notebook(cells, outputfile, execute=True, kernel='python3'): kernelspec = get_kernelspec(kernel) notebook = new_notebook(cells=cells, metadata={'language': 'python', 'kernelspec': kernelspec}) if execute: ep = ExecutePreprocessor(timeout=600, kernelname='python3') ep.preprocess(notebook, {'metadata': {'path': os.path.dirname(outputfile)}}) nbformat.write(notebook, outputfile)
def test_notebooks(tmpdir, filename): """ Runs a given notebook in a tmpdir """ import pylada from os.path import join, dirname from nbformat import read from nbconvert.preprocessors import ExecutePreprocessor from sys import version_info directory = join(dirname(pylada.__file__), 'notebooks') with open(join(directory, filename + ".ipynb")) as notebook_file: notebook = read(notebook_file, as_version=4) preprocessor = ExecutePreprocessor(kernel_name='python%i' % version_info.major) preprocessor.preprocess(notebook, {'metadata': {'path': str(tmpdir)}})
def run_notebook(notebook): # See http://nbconvert.readthedocs.io/en/latest/execute_api.html # TODO: Specify 'django_extensions' kernel and make it work on the server. # The kernel can be set as follows: # ep = ExecutePreprocessor(timeout=120, kernel_name='django_extensions') # This works locally, but on server, I wasn't able to create the kernel # (list available kernels by `jupyter kernelspec list`). # Default kernel currently works, given the `path` (directory from where to # execute the notebook) is set to //backend. It may fail if some Django # features are used in the notebook, but I haven't explored this. ep = ExecutePreprocessor(timeout=120) path = os.path.join(settings.REPO_DIR, 'backend') ep.preprocess(notebook, {'metadata': {'path': path}})
def translate(self): visitor = NBTranslator(self.document, self.app, self.docpath) self.document.walkabout(visitor) nb = _finilize_markdown_cells(visitor.nb) if self.app.config.nbexport_execute: ep = ExecutePreprocessor(allow_errors=True) try: ep.preprocess(nb, {'metadata': {}}) except CellExecutionError as e: self.app.warn(str(e)) self.output = nbformat.writes(nb)
def execute_notebook(notebook_path: str): with open(notebook_path, encoding='utf-8') as notebook: contents = nbformat.read(notebook, as_version=4) execution_processor = ExecutePreprocessor(timeout=60, kernel_name="python3") try: # Actually execute the notebook in the current working directory. execution_processor.preprocess(contents, {'metadata': {'path': os.getcwd()}}) return True except CellExecutionError: # This is a big chunk of JSON, but the stack trace makes it reasonably # clear which cell the error occurred in, so fixing it by actually # running the notebook will probably be easier. print(contents) return False
def execute_notebook(npth, dpth, timeout=1200, kernel='python3'): """ Execute the notebook at `npth` using `dpth` as the execution directory. The execution timeout and kernel are `timeout` and `kernel` respectively. """ ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) nb = nbformat.read(npth, as_version=4) t0 = timer() ep.preprocess(nb, {'metadata': {'path': dpth}}) t1 = timer() with open(npth, 'wt') as f: nbformat.write(nb, f) return t1 - t0
def test_tutorial_notebook(): pytest.importorskip('nbformat') pytest.importorskip('nbconvert') pytest.importorskip('matplotlib') import nbformat from nbconvert.preprocessors import ExecutePreprocessor rootdir = os.path.join(aospy.__path__[0], 'examples') with open(os.path.join(rootdir, 'tutorial.ipynb')) as nb_file: notebook = nbformat.read(nb_file, as_version=nbformat.NO_CONVERT) kernel_name = 'python' + str(sys.version[0]) ep = ExecutePreprocessor(kernel_name=kernel_name) with warnings.catch_warnings(record=True): ep.preprocess(notebook, {})
def run(self): f = { 'docdir': setup.confdir, 'builddir': setup.app.builder.outdir, 'nbname': self.arguments[0], } f['nbpath'] = "{docdir}/../examples/{nbname}.ipynb".format(**f) f['destdir'] = "{builddir}/examples/{nbname}".format(**f) if not os.path.exists(f['destdir']): os.makedirs(f['destdir']) f['uneval'] = "{destdir}/{nbname}.ipynb".format(**f) f['eval'] = "{destdir}/{nbname}.eval.ipynb".format(**f) f['py'] = "{destdir}/{nbname}.py".format(**f) # 1. Uneval notebook shutil.copyfile(f['nbpath'], f['uneval']) with open(f['nbpath']) as nb_f: nb = nbformat.read(nb_f, as_version=4) # 2. Python export_python(nb, f['py']) # 3. HTML (execute first) executer = ExecutePreprocessor(timeout=240) executer.preprocess(nb, {}) html = export_html(nb, f) # 4. Eval'd notebook with open(f['eval'], 'w') as eval_f: nbformat.write(nb, eval_f) # Create link to notebook and script files link_rst = "({uneval}; {eval}; {py})".format( uneval=formatted_link(f['uneval']), eval=formatted_link(f['eval']), py=formatted_link(f['py']), ) rst_file = self.state_machine.document.attributes['source'] self.state_machine.insert_input([link_rst], rst_file) # create notebook node nb_node = notebook_node('', html, format='html', source='nb_path') nb_node.source, nb_node.line = (self.state_machine .get_source_and_line(self.lineno)) # add dependency self.state.document.settings.record_dependencies.add(f['nbpath']) return [nb_node]
def execute_notebook(notebook_filename, notebook_filename_out, params_dict, run_path="", timeout=6000000): notebook_fp = os.path.join(run_path, notebook_filename) nb = read_in_notebook(notebook_fp) new_nb = set_parameters(nb, params_dict) ep = ExecutePreprocessor(timeout=timeout, kernel_name='python3') try: ep.preprocess(new_nb, {'metadata': {'path': run_path}}) except: msg = 'Error executing the notebook "{0}".\n\n'.format(notebook_filename) msg = '{0}See notebook "{1}" for the traceback.'.format(msg, notebook_filename_out) print(msg) raise finally: with open(notebook_filename_out, mode='wt') as f: nbformat.write(new_nb, f)
def run_notebook(notebook_name, nb_suffix='-out', out_path='.', timeout=3600, **execute_kwargs): """Runs a notebook and saves the output in the same notebook. Arguments: notebook_name (string): name of the notebook to be executed. nb_suffix (string): suffix to append to the file name of the executed notebook. timeout (int): timeout in seconds after which the execution is aborted. execute_kwargs (dict): additional arguments passed to `ExecutePreprocessor`. out_path (string): folder where to save the output notebook. """ timestamp_cell = "**Executed:** %s\n\n**Duration:** %d seconds." if str(notebook_name).endswith('.ipynb'): notebook_name = str(notebook_name)[:-len('.ipynb')] nb_name_input = notebook_name + '.ipynb' nb_name_output = notebook_name + '%s.ipynb' % nb_suffix nb_name_output = os.path.join(out_path, nb_name_output) print('- Executing: ', nb_name_input) execute_kwargs_ = dict(kernel_name = 'python%d' % sys.version_info[0]) if execute_kwargs is not None: execute_kwargs_.update(execute_kwargs) ep = ExecutePreprocessor(timeout=timeout, **execute_kwargs_) nb = nbformat.read(nb_name_input, as_version=4) start_time = time.time() try: # Execute the notebook ep.preprocess(nb, {'metadata': {'path': './'}}) except: # Execution failed, print a message then raise. msg = 'Error executing the notebook "%s".\n\n' % notebook_name msg += 'See notebook "%s" for the traceback.' % nb_name_output print(msg) raise else: # On successful execution, add timestamping cell duration = time.time() - start_time timestamp_cell = timestamp_cell % (time.ctime(start_time), duration) nb['cells'].insert(0, nbformat.v4.new_markdown_cell(timestamp_cell)) finally: # Save the notebook even when it raises an error nbformat.write(nb, nb_name_output) print('* Output: ', nb_name_output)
def test_tutorial_nb(file_path, workingdir, kernel=None): """Run tutorial jupyter notebook to catch any execution error. Parameters ---------- file_path : str path of tutorial .ipynb file workingdir: str path of the directory to run the tutorial in kernel: str Default None name of the kernel to use, if none, will use first kernel in the list """ tutorial_name = os.path.basename(file_path) sys.stdout.write('Testing {}...'.format(file_path)) sys.stdout.flush() tick = time.time() notebook = nbformat.read(file_path + '.ipynb', as_version=4) if kernel: eprocessor = ExecutePreprocessor(timeout=TIME_OUT, kernel_name=kernel) else: eprocessor = ExecutePreprocessor(timeout=TIME_OUT) success = True try: eprocessor.preprocess(notebook, {'metadata': {'path':workingdir}}) except Exception as err: err_msg = str(err) fail_dict[tutorial_name] = err_msg success = False finally: output_file = os.path.join(workingdir, "output.txt") output_nb = open(output_file, mode='w') nbformat.write(notebook, output_nb) output_nb.close() output_nb = open(output_file, mode='r') for line in output_nb: if "Warning:" in line: success = False if tutorial_name in fail_dict: fail_dict[tutorial_name] += "\n"+line else: fail_dict[tutorial_name] = "Warning:\n"+line sys.stdout.write(' Elapsed time: {0:.2f}s '.format(time.time()-tick )) sys.stdout.write(' [{}] \n'.format('Success' if success else 'Failed')) sys.stdout.flush()
def nbexec(nb_base, nb): base_name = nb_base.split('_')[1].split('.')[0].upper() afp_name = nb.split('_')[1].split('.')[0].upper() # Lee nb_base origen with io.open(nb_base, 'rt', encoding="utf8") as f: aux = nbformat.read(f, as_version=4) # Reemplaza afp_name u = aux['cells'][2]['source'].replace(base_name, afp_name) aux['cells'][2]['source'] = u # Opciones ep = ExecutePreprocessor(timeout=300) # Procesa aux (Revisar si preprocess modifica aux) ep.preprocess(aux, {}) # Escribe nb destino with io.open(nb, 'wt', encoding="utf8") as f: nbformat.write(aux, f) print "[INFO]--" + datetime.now().strftime('%Y-%M-%d %H:%M:%S') + "--" + "nbexec" + "--" + nb
def execute(self, write=True): """ Execute the specified notebook file, and optionally write out the executed notebook to a new file. Parameters ---------- write : bool, optional Write the executed notebook to a new file, or not. Returns ------- executed_nb_path : str, ``None`` The path to the executed notebook path, or ``None`` if ``write=False``. """ if path.exists(self._executed_nb_path) and not self.overwrite: logger.debug("Executed notebook already exists at {0}. Use " "overwrite=True or --overwrite (at cmd line) to re-run" .format(self._executed_nb_path)) return self._executed_nb_path # Execute the notebook logger.debug('Executing notebook using kwargs ' '"{}"...'.format(self._execute_kwargs)) executor = ExecutePreprocessor(**self._execute_kwargs) with open(self.nb_path) as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) try: executor.preprocess(nb, {'metadata': {'path': self.path_only}}) except CellExecutionError: # TODO: should we fail fast and raise, or record all errors? raise if write: logger.debug('Writing executed notebook to file {0}...' .format(self._executed_nb_path)) with open(self._executed_nb_path, 'w') as f: nbformat.write(nb, f) return self._executed_nb_path
def test_tutorial_nb(file_path): """Run tutorial jupyter notebook to catch any execution error. Parameters ---------- file_path : str path of tutorial markdown file """ tutorial_name = os.path.basename(file_path) notebook = nbformat.read(file_path + '_python.ipynb', as_version=4) eprocessor = ExecutePreprocessor(timeout=900) try: eprocessor.preprocess(notebook, {'metadata': {}}) except Exception as err: err_msg = "Python script successfully run without error or warning " \ "but notebook returned error:\n%s\nSomething weird happened." \ % (str(err)) fail_dict[tutorial_name] = err_msg
def run_notebook(filename): run_path = os.path.split(filename)[0] with open(filename) as f: nb = nbformat.read(f, as_version=4) try: ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(nb, {'metadata': {'path': run_path}}) # FIXME: use tempfile and mv to avoid interruption # better split the source code of the notebook and the compiled targets. with open(filename, 'wt') as f: nbformat.write(nb, f) except Exception as e: print('processing', filename, e)
def run_notebook(self, nb, f): """Runs a loaded notebook file.""" if PYTHON_MAJOR_VERSION == 3: kernel_name = 'python3' elif PYTHON_MAJOR_VERSION == 2: kernel_name = 'python2' else: raise Exception('Only Python 2 and 3 are supported') ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) try: ep.preprocess(nb, {'metadata': {'path': '.'}}) except CellExecutionError: msg = 'Error executing the notebook "%s".\n\n' % f.name msg += 'See notebook "%s" for the traceback.' % f.name print(msg) raise finally: nbformat.write(nb, f)
def test_notebook(notebook): fullfile = os.path.abspath(notebook) _, filename = os.path.split(fullfile) filename, _ = os.path.splitext(filename) if filename in KNOWN_FAILURES: pytest.skip('{0} is known to fail'.format(filename)) if filename in RPY2_NOTEBOOKS and not HAS_RPY2: pytest.skip('{0} since rpy2 is not installed'.format(filename)) if filename in JOBLIB_NOTEBOOKS and not JOBLIB_NOTEBOOKS: pytest.skip('{0} since joblib is not installed'.format(filename)) with io.open(fullfile, encoding='utf-8') as fp: nb = nbformat.read(fp, as_version=4) ep = ExecutePreprocessor(allow_errors=False, timeout=20, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': NOTEBOOK_DIR}})
def execute_ipynb(file_paths, exit_on_first, verbose): failed = False for test in file_paths: path = pathlib.Path(test) print() print("=" * 40) print(test) with open(test) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name="python3", store_widget_state=True) try: ep.preprocess(nb, {"metadata": {"path": path.parent}}) except CellExecutionError: failed = True with atomic_write(test, mode="w") as f: nbformat.write(nb, f) if failed and exit_on_first: raise SystemExit( f"notebook execution failed in {test}, error saved " "in notebook")
def notebook_execute(notebook_filename, kernel_name='python3', timeout=600, execution_dir='./', output_filemark="", output_timestamp=True): import nbformat from nbconvert.preprocessors import ExecutePreprocessor if output_timestamp: output_filemark += time.strftime(".%Y.%m.%d-%H.%M") executed_notebook_filename = "{1}{0}{2}".format( output_filemark, *os.path.splitext(notebook_filename)) with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name) ep.preprocess(nb, {'metadata': {'path': execution_dir}}) with open(executed_notebook_filename, 'wt') as f: nbformat.write(nb, f)
def test_invalid_notebooks( invalid_notebook_path, cell_location, error_name, error_value, error_output_type ): notebook_filename = script_relative_path(invalid_notebook_path) with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name="python3") try: ep.preprocess( nb, { "metadata": { "path": script_relative_path( notebook_filename[: notebook_filename.rfind("/")] ) } }, ) except CellExecutionError: error_message = get_dict_value(nb, cell_location) assert error_message.ename == error_name assert bool(re.search(error_value, error_message.evalue)) assert error_message.output_type == error_output_type
def test_notebooks(self, n): nb_path = pathlib.PurePath(n) nb_rel_path = nb_path.relative_to(BOOK_DIR) os.chdir(str(nb_path.parent)) with open(n) as f: try: nb = nbformat.read(f, as_version=4) except json.read.NotJSONError: self.fail(f'Notebook is not valid JSON: {nb_rel_path}.\n') except json.decoder.JSONDecodeError: self.fail(f'Unable to parse notebook {nb_rel_path}.\n') ep = ExecutePreprocessor(timeout=int(TIMEOUT), kernel_name=KERNEL) try: ep.preprocess(nb, {'metadata': {'path': RUN_PATH}}) except CellExecutionError as e: self.fail(f'{nb_rel_path} failed due to exception.\n{e.traceback}') except TimeoutError: self.fail(f'Timeout executing the notebook {n}.\n') finally: # This is where we could optionally write the notebook to an output file # with open(n_out + '.ipynb', mode='wt') as f: # nbformat.write(nb, f) pass
def test_that_examples_run(self): cwd = os.getcwd() os.chdir(examplesdir) example_files = [ f for f in listdir(examplesdir) if isfile(join(examplesdir, f)) and f.endswith(".ipynb") and not f.startswith("test") ] flag = 0 failed_examples = [] for f in example_files: nb_name, file_extension = os.path.splitext(os.path.basename(f)) with open(f) as file: nb = nbformat.read(file, as_version=4) nb.metadata.get('kernelspec', {})['name'] = kernel_name proc = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) proc.allow_errors = True proc.preprocess(nb) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) print(f, len(errors)) if errors: failed_examples.append(f) flag = 1 os.chdir(cwd) if len(failed_examples) > 0: print("failed examples: {0}".format(failed_examples)) self.assertEqual(flag, 0)
def test_notebook(path): import nbformat from nbconvert.preprocessors import ExecutePreprocessor if path in SKIP: pytest.skip("Notebook marked as 'skip'") if path in MARS: if not os.path.exists(os.path.expanduser("~/.ecmwfapirc")): pytest.skip("No ~/.ecmwfapirc") if path in CDS: if not os.path.exists(os.path.expanduser("~/.cdsapirc")): pytest.skip("No ~/.cdsapirc") # if path in TENSORFLOW: # if sys.version_info >= (3, 9): # pytest.skip("Tensorflow not yet ready on 3.9") with open(os.path.join(EXAMPLES, path)) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=60 * 60 * 5, kernel_name="python3") proc.preprocess(nb, {"metadata": {"path": EXAMPLES}})
def test_sd_roi(hdf5_ds_2, tmpdir_factory, lt_ctx, local_cluster_url): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'tcp', 'address': local_cluster_url}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) roi_params = {"shape": "rect", "x": 1, "y": 2, "width": 6, "height": 6} analysis = [{ "analysisType": 'SD_FRAMES', "parameters": { "roi": roi_params } }] notebook = notebook_generator(conn, dataset, analysis, save=True) notebook = io.StringIO(notebook.getvalue()) nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(timeout=600) ep.preprocess(nb, {"metadata": {"path": datadir}}) data_path = os.path.join(datadir, 'sd_result.npy') results = np.load(data_path) nx, ny = hdf5_ds_2.shape.nav roi = masks.rectangular(X=roi_params["x"], Y=roi_params["y"], Width=roi_params["width"], Height=roi_params["height"], imageSizeX=nx, imageSizeY=ny) udf = StdDevUDF() expected = lt_ctx.run_udf(dataset=hdf5_ds_2, udf=udf, roi=roi) assert np.allclose( results, expected['varsum'].raw_data, )
def run_notebook(notebook_path): # From https://www.blog.pythonlibrary.org/2018/10/16/testing-jupyter-notebooks/ nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name="python3") proc.allow_errors = True proc.preprocess(nb, {}) output_path = os.path.join(dirname, "{}_all_output.ipynb".format(nb_name)) with open(output_path, mode="wt") as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output) os.remove(output_path) return nb, errors
def run_notebook(notebook_path): with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name="python3") proc.allow_errors = True script_path = os.path.dirname(os.path.abspath(__file__)) package_path = os.path.abspath(os.path.join(script_path, "..", "..")) # Add shutdown for show method. shutdown_cell = new_code_cell( "from interpret import shutdown_show_server\nshutdown_show_server()") nb.cells.append(shutdown_cell) proc.preprocess(nb, {"metadata": {"path": package_path}}) errors = [] for cell in nb.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output) return nb, errors
def run_notebook(notebook_path, timeout=hours(6), save_nb_path=None): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=timeout, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': dirname}}) if save_nb_path: with open(save_nb_path, mode='wt') as f: nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def run_notebook(notebook_path): nb_name, _ = os.path.splitext(os.path.basename(notebook_path)) dirname = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) proc = ExecutePreprocessor(timeout=600, kernel_name='python3') proc.allow_errors = True proc.preprocess(nb, {'metadata': {'path': './'}}) #output_path = os.path.join(dirname, '{}_all_output.ipynb'.format(nb_name)) #with open(output_path, mode='wt') as f: # nbformat.write(nb, f) errors = [] for cell in nb.cells: if 'outputs' in cell: for output in cell['outputs']: if output.output_type == 'error': errors.append(output) return nb, errors
def runtest(self): self._skip() with io.open(self.name,encoding='utf8') as nb: notebook = nbformat.read(nb, as_version=4) # TODO: which kernel? run in pytest's or use new one (make it option) _timeout = self.parent.parent.config.getini('nbsmoke_cell_timeout') kwargs = dict(timeout=int(_timeout) if _timeout!='' else 300, allow_errors=False, # or sys.version_info[1] ? kernel_name='python') ep = ExecutePreprocessor(**kwargs) with cwd(os.path.dirname(self.name)): # jupyter notebook always does this, right? ep.preprocess(notebook,{}) # TODO: clean up this option handling if self.parent.parent.config.option.store_html != '': he = nbconvert.HTMLExporter() # could maybe use this for chance of testing the html? but not the aim of this project #he.template_file = 'basic' html, resources = he.from_notebook_node(notebook) with io.open(os.path.join(self.parent.parent.config.option.store_html,os.path.basename(self.name)+'.html'),'w',encoding='utf8') as f: f.write(html)
def execute(path_nb, dated=True, timeout=600, kernel_name='python3', verbose=False): """ """ folder = os.path.dirname(path_nb) filename = os.path.basename(path_nb) now = dt.datetime.now().strftime('%Y%m%d_%H%M%S.%f')[:-3] if dated: save_name = '{}_{}'.format(now, filename) else: save_name = filename # path_output path_out = os.path.join(folder, save_name) # load nb with open(path_nb, 'r', encoding='utf-8') as f: nb = nbformat.read(f, as_version=4) # create ep ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel_name) # run catching errors try: with warnings.catch_warnings(): warnings.simplefilter("ignore") out = ep.preprocess(nb, {'metadata': {'path': '.'}}) except CellExecutionError: out = None msg = 'Error executing notebook {}\nSee notebook {} for traceback.' msg = msg.format(path_nb, path_out) print(msg) raise finally: with open(path_out, mode='w', encoding='utf-8') as f: nbformat.write(nb, f) if verbose: md = 'Notebook **{}** run and saved as **{}**' md = md.format(path_nb, path_out) display(Markdown(md)) return path_out, now
def run_notebook( *, notebook_name=None, notebook_path=None, output_notebook_name=None, output_notebook_path=None, timeout=-1, notebook_version=4, kernel='python3', ): """Execute a jupyter notebook kernel name is an issue: https://github.com/jupyter/nbconvert/issues/515 """ if notebook_path is None: notebook_path = paths['notebook_path'] else: notebook_path = pathlib.Path(notebook_path) if output_notebook_path is None: output_notebook_path = paths['interim_data_path'] else: output_notebook_path = pathlib.Path(output_notebook_path) if output_notebook_name is None: output_notebook_name = notebook_name output_notebook_fq = output_notebook_path / output_notebook_name with open(notebook_path / notebook_name) as f: nb = nbformat.read(f, as_version=notebook_version) ep = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) try: out = ep.preprocess(nb, {'metadata': {'path': notebook_path}}) except CellExecutionError: out = None msg = f"""Error executing the notebook "{notebook_name}". See notebook "{str(output_notebook_fq)}" for the traceback.' """ logger.error(msg) raise finally: with open(output_notebook_fq, mode='w', encoding='utf-8') as f: nbformat.write(nb, f) return output_notebook_name
def run_notebook(nb_wd, nb_path): """Runs a given notebook and saves it Executes the passed notebook with nb_wd as the working directory. If an error occurs during the execution, a message is raised to the user and the notebook is saved anyway, including the traceback. Parameters ---------- nb_wd : Path or str Path to the folder which should be used as a working directory for the execution of the notebook nb_path : Path or str Full path to the notebook which should be run. Returns ------- Nothing """ if not isinstance(nb_wd, Path): nb_wd = Path(nb_wd) if not isinstance(nb_path, Path): nb_path = Path(nb_path) with nb_path.open() as f: nb = nbformat.read(f, as_version=nbformat.NO_CONVERT) # Configure notebook execution mode # Timeout = None means no restriction on runtime of cells ep = ExecutePreprocessor(timeout=None, kernel_name='python3') # The code for the following error handling is taken from the # official nbconvert documentation: # https://nbconvert.readthedocs.io/en/latest/execute_api.html try: # Run notebook out = ep.preprocess(nb, {'metadata': {'path': str(nb_wd)}}) except CellExecutionError: out = None msg = f'Error executing the notebook "{str(nb_path)}".\n\n' msg += 'See the notebook for the traceback.\n' print(msg) raise finally: # Save it. Includes tracebacks should an error have occured. with nb_path.open('wt') as f: nbformat.write(nb, f) return
def execute_notebook(notebook_file_path: str, output_file_folder: str): file_name = os.path.basename(os.path.normpath(notebook_file_path)) # Read notebook with open(notebook_file_path) as f: nb = nbformat.read(f, as_version=4) has_error = False # Execute notebook try: # Create preprocessors remove_no_execute_cells_preprocessor = RemoveNoExecuteCells() execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3") # Use no-execute preprocessor ( nb_no_execute_cells_removed, resources_no_execute_cells_removed, ) = remove_no_execute_cells_preprocessor.preprocess(nb) # Execute notebook out = execute_preprocessor.preprocess( nb_no_execute_cells_removed, resources_no_execute_cells_removed ) except Exception as error: out = None print(f"Error executing the notebook: {notebook_file_path}.\n\n") has_error = True raise finally: output_file_path = os.path.join( output_file_folder, "failure" if has_error else "success", file_name ) # Create directories if they don't exist if not os.path.exists(os.path.dirname(output_file_path)): try: os.makedirs(os.path.dirname(output_file_path)) except OSError as exc: # Guard against race condition if exc.errno != errno.EEXIST: raise print(f"Writing output to: {output_file_path}") with open(output_file_path, mode="w", encoding="utf-8") as f: nbformat.write(nb, f)
def test_func(self): passing = True print("\n--------------- Testing {0} ---------------".format(nbname)) print(" {0}".format(nbpath)) if nbname in py2Ignore and sys.version_info[0] == 2: print(" Skipping {}".format(nbname)) return ep = ClearOutputPreprocessor() with open(nbpath) as f: nb = nbformat.read(f, as_version=4) ep.preprocess(nb, {}) ex = ExecutePreprocessor(timeout=600, kernel_name='python{}'.format( sys.version_info[0]), allow_errors=True) out = ex.preprocess(nb, {}) for cell in out[0]['cells']: if 'outputs' in cell.keys(): for output in cell['outputs']: if output[ 'output_type'] == 'error': #in output.keys(): passing = False err_msg = [] for o in output['traceback']: err_msg += ["{}".format(o)] err_msg = "\n".join(err_msg) msg = """ \n <<<<< {} FAILED >>>>> \n {} in cell [{}] \n-----------\n{}\n-----------\n ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format(nbname, output['ename'], cell['execution_count'], cell['source'], err_msg) assert passing, msg print("\n ..... {0} Passed ..... \n".format(nbname))
def nb_task(ds, **kwargs): notebook_dir = os.path.dirname(notebook_path) with open(notebook_path) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=21600) try: out = ep.preprocess(nb, {'metadata': {'path': notebook_dir}}) except CellExecutionError: msg = 'Error executing the notebook "%s".\n\n' % notebook_path msg += 'See notebook "%s" for the traceback.' % notebook_path print(msg) raise finally: with open(notebook_path, mode='wt') as f: nbformat.write(nb, f)
def exec_notebook(self, nb, resources, nb_dir): nb_copy, _ = AddEpilog(config=self.config, pkg_dir=self.output_dir).preprocess(nb, resources) resources['outputs']['notebooks/executed-source.ipynb'] = nbformat.writes(nb).encode('utf-8') exec_nb_path = self.write_notebook(nb_copy) try: ep = ExecutePreprocessor(config=self.config) nb, _ = ep.preprocess(nb, {'metadata': {'path': nb_dir}}) except CellExecutionError as e: raise CellExecutionError("Errors executing noteboook. See output at {} for details.\n{}" .format(exec_nb_path, '')) return nb, resources
def test_monitor_cell(self): expected_logs = [ "Beginning machine learning...", "Still training...", "Done!", # Handle unicode "хнЧ" ] with open("./tests/jupyter_test_file.py.ipynb") as f: nb = nbformat.read(f, as_version=4.0) ep = ExecutePreprocessor(timeout=5000, kernel_name="python") result = ep.preprocess(nb, {}) # Accumulate all output from cell 2 all_stdout = "" second_cell_output = result[0]["cells"][1]["outputs"] for output in second_cell_output: if output.name == "stdout": all_stdout = all_stdout + output.text # Assert on all output from cell2 for log in expected_logs: if PY2: assert_in(log, all_stdout.encode("utf-8")) continue assert_in(log, all_stdout) # Verify that variables declared in previous cells can be affected third_cell_output = result[0]["cells"][2]["outputs"] assert third_cell_output[0].text == "a=1\n" # Make sure logs were persisted log_dir = get_hyperdash_logs_home_path_for_job("test_jupyter") latest_log_file = max([ os.path.join(log_dir, filename) for filename in os.listdir(log_dir) ], key=os.path.getmtime) with open(latest_log_file, 'r') as log_file: data = log_file.read() for log in expected_logs: assert_in(log, data) os.remove(latest_log_file)
def grade_notebook(path, notebook_path, output_path, kernal='python3'): print("Running notebooks: ", notebook_path) with open(notebook_path) as f: nb = nbf.read(f, as_version=4) ep = ExecutePreprocessor(allow_errors=True,timeout=600, kernel_name=kernal) try: out = ep.preprocess(nb, {'metadata': {'path': path}}) except CellExecutionError: out = None msg = 'Error executing the notebook "%s".\n\n' % notebook_filename msg += 'See notebook "%s" for the traceback.' % output_path print(msg) raise finally: print("Writing to", output_path) with open( output_path, mode='w', encoding='utf-8') as f: nbf.write(nb, f) return 4, "4. Graded."
def retrieve_errors(nb_path): """ Find all errors that occur when running a nb """ with nb_path.open() as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=True) out, _ = ep.preprocess(nb, {'metadata': {'path': '.'}}) errors = [] for cell in out.cells: if "outputs" in cell: for output in cell["outputs"]: if output.output_type == "error": errors.append(output.evalue) return errors
def do_execute(self, script, kernel, timeout=600): cells = [nbformat.v4.new_code_cell(source=script)] nb = nbformat.v4.new_notebook(cells=cells, metadata={'language': 'python'}) # Execute the script executor = ExecutePreprocessor(timeout=timeout, kernel_name=kernel) nb, metadata = executor.preprocess(nb, {'metadata': {}}) # Export html from the notebook html_exporter = HTMLExporter() html_exporter.template_file = 'basic' body, _ = html_exporter.from_notebook_node(nb) # Extract the result part of the html output = ScriptExecutor.extract_output(body, kernel) return output
def test_radial_fourier_default(hdf5_ds_2, tmpdir_factory, lt_ctx): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'local'}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) analysis = [{ "analysisType": "RADIAL_FOURIER", "parameters": { 'shape': 'radial_fourier', 'cx': 0, 'cy': 0, 'ri': 0, 'ro': 2, 'n_bins': 2, 'max_order': 23, } }] notebook = notebook_generator(conn, dataset, analysis, save=True) notebook = io.StringIO(notebook.getvalue()) nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(timeout=600) out = ep.preprocess(nb, {"metadata": {"path": datadir}}) channels = ["dominant_0", "absolute_0_0", "absolute_0_1"] results = {} for channel in channels: data_path = os.path.join(datadir, f"radial_result_{channel}.npy") results[channel] = np.load(data_path) analysis = lt_ctx.create_radial_fourier_analysis(dataset=hdf5_ds_2, cx=0, cy=0, ri=0, ro=2, n_bins=2, max_order=23) expected = lt_ctx.run(analysis) assert np.allclose(results["dominant_0"], expected["dominant_0"].raw_data) assert np.allclose(results["absolute_0_0"], expected["absolute_0_0"].raw_data) assert np.allclose(results["absolute_0_1"], expected["absolute_0_1"].raw_data)
def test_sum_default(hdf5_ds_2, tmpdir_factory): datadir = tmpdir_factory.mktemp('template_tests') conn = {'connection': {'type': 'local'}} path = hdf5_ds_2.path dataset = _get_hdf5_params(path) analysis = [{"analysisType": "SUM_FRAMES", "parameters": {"roi": {}}}] notebook = notebook_generator(conn, dataset, analysis, save=True) notebook = io.StringIO(notebook.getvalue()) nb = nbformat.read(notebook, as_version=4) ep = ExecutePreprocessor(timeout=600) out = ep.preprocess(nb, {"metadata": {"path": datadir}}) data_path = os.path.join(datadir, 'sum_result.npy') result = np.load(data_path) with hdf5_ds_2.get_reader().get_h5ds() as h5ds: data = h5ds[:] expected = data.sum(axis=(0, 1)) assert np.allclose(expected, result)
def generate_html(self, kernel_name=""): nb_executor = ExecutePreprocessor(kernel_name=kernel_name) nb_executor.allow_errors = True html_exporter = HTMLExporter() html_writer = FilesWriter() with open(self.output_ipynb_path) as f: nb = nbformat.read(f, as_version=4) nb = nb_executor.preprocess(nb)[0] body, resources = html_exporter.from_notebook_node(nb) # HTML writer automatically adds .html to the end, so get rid of it html_writer.write( body, resources, self.output_html_path.replace(".html", "") ) print(f"Written file to {self.output_html_path}") if self.verbose else None self.metadata["html-file"] = os.path.basename(self.output_html_path) self.metadata.mark_as_updated()
def execute_notebook(src='estimate.src.ipynb', dest='estimate.ipynb'): """Executes the analysis notebook and writes out a copy with all of the resulting tables and plots. Parameters ---------- src: str, optional Source notebook to execute dest: str, optional Output notebook """ with open(src) as fp: nb = nbformat.read(fp, 4) exp = ExecutePreprocessor(timeout=300) updated_nb, _ = exp.preprocess(nb, {}) with open(dest, 'w') as fp: nbformat.write(updated_nb, fp)
def execute_notebook(input_file: Path, output_file: Path) -> Dict: ''' Execute notebook within a folder Parameters ---------- input_file notebook file to be executed output_file notebook execution output file Returns ------- a pandas dataframe contains filename and error(s) encountered. ''' error = {} with open(input_file) as f: try: nb = nbformat.read(f, as_version=4) kernel = nb.dict()['metadata']['kernelspec']['name'] logger.info(f'Running... {input_file.stem}') ep = ExecutePreprocessor(timeout=600, kernel_name=kernel) out = ep.preprocess(nb, {'metadata': {'path': input_file.parent}}) except CellExecutionError: out = None msg = f'Error executing notebook".\n' msg += f'See notebook "{output_file}" for the traceback.' logger.info(msg) error = {'error': msg, 'file': input_file} finally: with open(output_file, mode='w', encoding='utf-8') as f: nbformat.write(nb, f) return error
def execute(nb_in_fn, kernel_name='python3', run_path='.'): nb_out_fn = nb_in_fn.replace('.ipynb', '.nbconvert.ipynb') with open(nb_in_fn) as f: nb = nbformat.read(f, as_version=4) ep = ExecutePreprocessor(timeout=600, kernel_name=kernel_name) failed = False try: out = ep.preprocess(nb, {'metadata': {'path': run_path}}) except CellExecutionError as e: out = None print('Error executing the notebook "{}". Traceback:'.format(nb_in_fn)) print(e.traceback) failed = True finally: with open(nb_out_fn, mode='wt') as f: nbformat.write(nb, f) return not failed
def run_notebook_with_args(input_notebook, output_notebook, unknown): with codecs.open(input_notebook, encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) orig_parameters = extract_parameters(nb) params = [] args = compute_args(unknown) for k, v in args.items(): for p in orig_parameters: if p.name == k: params.append(p.with_value(v)) if len(params) > 0: nb = replace_definitions(nb, params, execute=False) ep = ExecutePreprocessor(timeout=None, kernel_name='python3') out = ep.preprocess(nb, {'metadata': {'path': '.'}}) with codecs.open(output_notebook, encoding="utf-8", mode="w") as f: nbformat.write(nb, f)