def process_notebook(notebook_filename, execute=True): '''Checks if an IPython notebook runs without error from start to finish. If so, writes the notebook to HTML (with outputs) and overwrites the .ipynb file (without outputs). ''' with open(notebook_filename) as f: nb = nbformat.read(f, as_version=4) clear = ClearOutputPreprocessor() try: if execute: # Check that the notebook runs ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(nb, {'metadata': {'path': ''}}) msg = '' except CellExecutionError: msg = f'\n Error executing the notebook "{notebook_filename}".\n' msg += f' See notebook "{notebook_filename}" for the traceback.' # Clear notebook outputs and save as .ipynb cleared = clear.preprocess(nb, {}) with open(notebook_filename, mode='w', encoding='utf-8') as f: nbformat.write(nb, f) print(f"Processed {notebook_filename}{msg}") return
def clear_notebooks(root): """Clear the outputs of documentation notebooks.""" preprocessor = ClearOutputPreprocessor() for dirpath, dirnames, filenames in os.walk(root): for filename in sorted(filenames): if os.path.splitext(filename)[1] == '.ipynb': # read in the notebook pth = os.path.join(dirpath, filename) with open(pth, 'r') as fh: orig_nb = read(fh, 4) # copy the original notebook new_nb = deepcopy(orig_nb) # check outputs of all the cells new_nb = preprocessor.preprocess(new_nb, {})[0] # clear metadata new_nb.metadata = {} # write the notebook back to disk with open(pth, 'w') as fh: write(new_nb, fh, 4)
def NB(notebook, test_notebook): working_dir = '.' with open(notebook, 'r') as f: nb = nbformat.read(f, as_version=4) with open(test_notebook, 'r') as f: test_nb = nbformat.read(f, as_version=4) test_start = len(nb.cells) nb.cells.extend(test_nb.cells) # run the notebook meta = dict(metadata=dict(path=working_dir)) p1 = ClearOutputPreprocessor() p2 = ExecutePreprocessor(allow_errors=True) p1.preprocess(nb, meta) p2.preprocess(nb, meta) try: output_notebook = os.path.join(os.environ['REPORT_DIR'], os.environ['DRIVER_NAME'] + '.ipynb') with open(output_notebook, 'w') as f: nbformat.write(nb, f) except: pass tests = [] for cell in nb.cells[test_start:]: if cell.cell_type == 'code': tests.append(CellTest(cell)) return tests
def clear_notebook(path, kwargs={ "SCOPETYPE": "OPENADC", "PLATFORM": "CWLITEARM", "VERSION": "HARDWARE" }): real_path = Path(path) body = "" with open(real_path, "r", encoding="utf-8") as nbfile: nb = nbformat.read(nbfile, as_version=4) # special case: if the top block isn't a parameter block, nbparameterise will: # * error out if invalid python syntax # * replace code with a parameter block (whatever we passed in with kwargs, including an empty block) # so if no parameters being changed, don't run nbparameterise if len(kwargs) > 0: orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, **kwargs) new_nb = replace_definitions(nb, params, execute=False) else: new_nb = nb co = ClearOutputPreprocessor() exporter = NotebookExporter() node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}}) body, resources = exporter.from_notebook_node(node, resources) with open(real_path, "w", encoding="utf-8") as nbfile: nbfile.write(body)
def run_notebook(self, name, event=None): """ run a given notebook immediately. the job parameter is the name of the job script as in ipynb. Inserts and returns the Metadata document for the job. """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute ep_kwargs = {'timeout': None} ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: if not meta_job.kind_meta.get('keep_output', False): resources = { } # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, {'metadata': {'path': '/'}}) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results
def clear_notebooks(root): """Clear the outputs of documentation notebooks.""" # cleanup ignored files run(['git', 'clean', '-fdX', root]) # remove release/autograded/feedback if os.path.exists(os.path.join(root, "user_guide", "release")): shutil.rmtree(os.path.join(root, "user_guide", "release")) if os.path.exists(os.path.join(root, "user_guide", "autograded")): shutil.rmtree(os.path.join(root, "user_guide", "autograded")) if os.path.exists(os.path.join(root, "user_guide", "feedback")): shutil.rmtree(os.path.join(root, "user_guide", "feedback")) if os.path.exists( os.path.join(root, "user_guide", "downloaded", "ps1", "extracted")): shutil.rmtree( os.path.join(root, "user_guide", "downloaded", "ps1", "extracted")) print("Clearing outputs of notebooks in '{}'...".format( os.path.abspath(root))) preprocessor = ClearOutputPreprocessor() for dirpath, dirnames, filenames in os.walk(root): is_submitted = _check_if_directory_in_path(dirpath, 'submitted') for filename in sorted(filenames): if os.path.splitext(filename)[1] == '.ipynb': # read in the notebook pth = os.path.join(dirpath, filename) with open(pth, 'r') as fh: orig_nb = read(fh, 4) # copy the original notebook new_nb = deepcopy(orig_nb) # check outputs of all the cells if not is_submitted: new_nb = preprocessor.preprocess(new_nb, {})[0] # clear metadata new_nb.metadata = { "kernelspec": { "display_name": "Python", "language": "python", "name": "python" } } # write the notebook back to disk with open(pth, 'w') as fh: write(new_nb, fh, 4) if orig_nb != new_nb: print("Cleared '{}'".format(pth))
def clear_notebook(path): real_path = Path(path) body = "" with open(real_path, "r", encoding="utf-8") as nbfile: nb = nbformat.read(nbfile, as_version=4) co = ClearOutputPreprocessor() exporter = NotebookExporter() node, resources = co.preprocess(nb, {'metadata': {'path': './'}}) body, resources = exporter.from_notebook_node(node, resources) with open(real_path, "w", encoding="utf-8") as nbfile: nbfile.write(body)
def clear(notebook_filename, nbconvert_filename): with open(notebook_filename, 'r') as fh: nb = nbformat.read(fh, 4) # check outputs of all the cells preprocessor = ClearOutputPreprocessor() clear_nb = preprocessor.preprocess(nb, {})[0] # clear metadata clear_nb.metadata = {} # write the notebook back to disk with open(nbconvert_filename, 'w') as fh: nbformat.write(clear_nb, fh, 4)
def test_func(self): passing = True print("\n--------------- Testing {0} ---------------".format(nbname)) print(" {0}".format(nbpath)) if nbname in py2Ignore and sys.version_info[0] == 2: print(" Skipping {}".format(nbname)) return ep = ClearOutputPreprocessor() with open(nbpath) as f: nb = nbformat.read(f, as_version=4) ep.preprocess(nb, {}) ex = ExecutePreprocessor( timeout=600, kernel_name='python{}'.format(sys.version_info[0]), allow_errors=True ) out = ex.preprocess(nb, {}) for cell in out[0]['cells']: if 'outputs' in cell.keys(): for output in cell['outputs']: if output['output_type'] == 'error': passing = False err_msg = [] for o in output['traceback']: err_msg += ["{}".format(o)] err_msg = "\n".join(err_msg) msg = """ \n <<<<< {} FAILED >>>>> \n {} in cell [{}] \n-----------\n{}\n-----------\n ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format( nbname, output['ename'], cell['execution_count'], cell['source'], err_msg ) assert passing, msg print("\n ..... {0} Passed ..... \n".format(nbname))
def execute(input, to_html, to_self, to_clear): print('Processing ' + Fore.LIGHTGREEN_EX + input) with open(input, 'rb') as f: notebook = nbformat.read(f, as_version=4) if to_html or to_self: print(" Executing") ep = ExecutePreprocessor(timeout=600, kernel_name='python3') ep.preprocess(notebook, {'metadata': {'path': './'}}) if to_html: html_output = input + ".html" print(" Saving to HTML") exportHTML = HTMLExporter() (body, resources) = exportHTML.from_notebook_node(notebook) with open(html_output, 'wt') as f: f.write(body) if to_clear: print(" Clearing") ep = ClearOutputPreprocessor() ep.preprocess(notebook, {'metadata': {'path': './'}}) if to_self or to_clear: stringio = io.StringIO() nbformat.write(notebook, stringio) self_output = input print(" Saving to self") with open(self_output, 'wb') as f: s = stringio.getvalue().encode('utf-8') f.write(s)
def clear_notebook(path): real_path = Path(path) body = "" with open(real_path, "r", encoding="utf-8") as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE="OPENADC", PLATFORM="CWLITEARM") new_nb = replace_definitions(nb, params, execute=False) co = ClearOutputPreprocessor() exporter = NotebookExporter() node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}}) body, resources = exporter.from_notebook_node(node, resources) with open(real_path, "w", encoding="utf-8") as nbfile: nbfile.write(body)
def executenb(nb, cwd=None, km=None, **kwargs): resources = {} if cwd is not None: resources['metadata'] = {'path': cwd} # pragma: no cover # Clear any stale output, in case of exception nb, resources = ClearOutputPreprocessor().preprocess(nb, resources) ep = ExecutePreprocessorWithOutputWidget(**kwargs) return ep.preprocess(nb, resources, km=km)[0]
def executenb(nb, cwd=None, km=None, **kwargs): resources = {} if cwd is not None: resources['metadata'] = {'path': cwd} # pragma: no cover # Clear any stale output, in case of exception nb, resources = ClearOutputPreprocessor().preprocess(nb, resources) executor = VoilaExecutor(nb, km=km, **kwargs) return executor.execute(nb, resources, km=km)
async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, resources = ClearOutputPreprocessor().preprocess( nb, {'metadata': { 'path': self.cwd }}) for cell_idx, input_cell in enumerate(nb.cells): try: task = asyncio.ensure_future( self.executor.execute_cell(input_cell, None, cell_idx, store_history=False)) while True: done, pending = await asyncio.wait( {task}, timeout=self.voila_configuration. http_keep_alive_timeout) if pending: # If not done within the timeout, we send a heartbeat # this is fundamentally to avoid browser/proxy read-timeouts, but # can be used in a template to give feedback to a user self.write("<script>voila_heartbeat()</script>\n") self.flush() continue output_cell = await task break except TimeoutError: output_cell = input_cell break except Exception as e: self.log.exception('Error at server while executing cell: %r', input_cell) output_cell = nbformat.v4.new_code_cell() if self.executor.should_strip_error(): output_cell.outputs = [{ "output_type": "stream", "name": "stderr", "text": "An exception occurred at the server (not the notebook). {}" .format(self.executor.cell_error_instruction), }] else: output_cell.outputs = [{ 'output_type': 'error', 'ename': type(e).__name__, 'evalue': str(e), 'traceback': traceback.format_exception(*sys.exc_info()), }] finally: yield output_cell
def clearNbOutput(): cwd = os.getcwd() # walk the test directory and find all notebooks for dirname, dirnames, filenames in os.walk(nbdir): if dirname != ".ipynb_checkpoints": for filename in filenames: if (filename.endswith(".ipynb") and not filename.endswith("-checkpoint.ipynb")): with open(os.path.join(dirname, filename)) as f: print("Clearing output from {}".format(filename)) nb = nbformat.read(f, as_version=4) ep = ClearOutputPreprocessor() ep.preprocess(nb, { "kernel_name": "python{}".format(sys.version_info[0]) }) print(" ... done\n") return True
def generate_exercises(): p = Path(*EXERCISES_DIR) exporter = HTMLExporter() exporter.register_preprocessor(ClearOutputPreprocessor(), enabled=True) for exercise in p.iterdir(): if exercise.suffix == '.ipynb': html, _ = exporter.from_file(exercise.open()) with open(exercise.with_suffix('.html').name, 'w') as f: f.write(html)
def unexecute(self): """ Unexecutes the notebook i.e. removes all output cells """ _logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}") if not self.executed_nb_path.exists(): _logger.warning( f"{self.executed_nb_path} not found, nothing to clean") return with open(self.executed_nb_path) as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) if nb['metadata'].get('docs_executed', None): nb['metadata'].pop('docs_executed') clear_executor = ClearOutputPreprocessor() clear_executor.preprocess(nb, {}) with open(self.executed_nb_path, 'w') as f: nbformat.write(nb, f)
def clear_notebook(path): """Clears output from a notebook Parameters ---------- path: str (optional) path to a Jupyter Notebook Returns ------- None """ with open(_get_path(path), encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) pp = ClearOutputPreprocessor() pp.preprocess(nb, {}) with open(path, "w", encoding="utf-8") as f: nbformat.write(nb, f)
def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" km = self.kernel_manager.get_kernel(kernel_id) nb, resources = ClearOutputPreprocessor().preprocess(nb, {'metadata': {'path': self.cwd}}) ep = VoilaExecutePreprocessor(config=self.traitlet_config) with ep.setup_preprocessor(nb, resources, km=km): for cell_idx, cell in enumerate(nb.cells): res = ep.preprocess_cell(cell, resources, cell_idx, store_history=False) yield res[0]
async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, _ = ClearOutputPreprocessor().preprocess( nb, {'metadata': { 'path': self.cwd }}) for cell_idx, input_cell in enumerate(nb.cells): try: output_cell = await self.executor.execute_cell( input_cell, None, cell_idx, store_history=False) except TimeoutError: output_cell = input_cell break except CellExecutionError: self.log.exception('Error at server while executing cell: %r', input_cell) if self.executor.should_strip_error(): strip_code_cell_warnings(input_cell) self.executor.strip_code_cell_errors(input_cell) output_cell = input_cell break except Exception as e: self.log.exception('Error at server while executing cell: %r', input_cell) output_cell = nbformat.v4.new_code_cell() if self.executor.should_strip_error(): output_cell.outputs = [{ 'output_type': 'stream', 'name': 'stderr', 'text': 'An exception occurred at the server (not the notebook). {}' .format(self.executor.cell_error_instruction), }] else: output_cell.outputs = [{ 'output_type': 'error', 'ename': type(e).__name__, 'evalue': str(e), 'traceback': traceback.format_exception(*sys.exc_info()), }] finally: yield output_cell await self._cleanup_resources()
def nb_exec(fname, name): ver = make_version(name) dirpath = os.path.join(data_dir, name, ver) execpath = os.path.join(dirpath, 'src') src_nb = os.path.join(execpath, 'src.ipynb') ensure_dir(dirpath) ensure_dir(execpath) logger.info(f'Copying {fname} to {src_nb}') shutil.copy2(fname, src_nb) with open(src_nb) as f: nb = nbformat.read(f, as_version=4) co = ClearOutputPreprocessor() nb, _ = co.preprocess(nb, {}) ep = ExecutePreprocessor(timeout=-1, kernel_name='python3') try: ep.preprocess(nb, {'metadata': {'path': execpath}}) except CellExecutionError as e: print(e) with open(os.path.join(dirpath, 'error.log'), 'w') as f: f.write(str(e)) print('Stack trace saved') with open(os.path.join(dirpath, 'output.ipynb'), 'w', encoding='utf-8') as f: nbformat.write(nb, f) exp = HTMLExporter() body, res = exp.from_notebook_node(nb) with open(os.path.join(dirpath, 'output.html'), 'w', encoding='utf-8') as f: f.write(body)
def clear_notebooks(root): """Clear the outputs of documentation notebooks.""" # cleanup ignored files run(['git', 'clean', '-fdX', root]) print("Clearing outputs of notebooks in '{}'...".format( os.path.abspath(root))) preprocessor = ClearOutputPreprocessor() for dirpath, dirnames, filenames in os.walk(root): is_submitted = _check_if_directory_in_path(dirpath, 'submitted') for filename in sorted(filenames): if os.path.splitext(filename)[1] == '.ipynb': # read in the notebook pth = os.path.join(dirpath, filename) with open(pth, 'r') as fh: orig_nb = read(fh, 4) # copy the original notebook new_nb = deepcopy(orig_nb) # check outputs of all the cells if not is_submitted: new_nb = preprocessor.preprocess(new_nb, {})[0] # clear metadata new_nb.metadata = {} # write the notebook back to disk with open(pth, 'w') as fh: write(new_nb, fh, 4) if orig_nb != new_nb: print("Cleared '{}'".format(pth))
def unexecute(self, remove_gh=False): """ Unexecutes the notebook i.e. removes all output cells. If remove_gh=True looks to see if notebook metadata contains an executed tag. If it doesn't it means the notebook either errored or was not run (for case when only specific notebooks chosen to build examples) and removes the notebooks so old ones can be used """ _logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}") if not self.executed_nb_path.exists(): _logger.warning( f"{self.executed_nb_path} not found, nothing to clean") return with open(self.executed_nb_path, encoding='utf-8') as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) if not remove_gh: if nb['metadata'].get('docs_executed', None): nb['metadata'].pop('docs_executed') clear_executor = ClearOutputPreprocessor() clear_executor.preprocess(nb, {}) with open(self.executed_nb_path, 'w', encoding='utf-8') as f: nbformat.write(nb, f) elif remove_gh: executed_flag = nb['metadata'].get('docs_executed', None) if executed_flag != 'executed': _logger.warning(f"Notebook {self.nb} not executed or errored, " f"version already on website will be used") os.remove(self.executed_nb_path) os.remove(self.output_path.joinpath(self.nb_name + '.html')) else: _logger.info(f"Notebook {self.nb} executed, " f"new version will be uploaded to website") clear_executor = ClearOutputPreprocessor() clear_executor.preprocess(nb, {}) with open(self.executed_nb_path, 'w', encoding='utf-8') as f: nbformat.write(nb, f)
async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, resources = ClearOutputPreprocessor().preprocess( nb, {'metadata': { 'path': self.cwd }}) stop_execution = False for cell_idx, cell in enumerate(nb.cells): if stop_execution: break try: res = await self.executor.execute_cell(cell, None, cell_idx, store_history=False) except TimeoutError: res = cell stop_execution = True yield res
def clean(): _delete_generated_files() config = Config() config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()] exporter = NotebookExporter(config=config) for notebook in get_files("ipynb"): with open(notebook, encoding="utf8") as nb_file: nb = nbformat.read(nb_file, as_version=4) if not nb.cells[-1]["source"]: nb.cells.pop() format_code_cells(nb) for cell_id, cell in enumerate(nb.cells): cell["id"] = f"{notebook.stem}-{cell_id}" ipynb, _ = exporter.from_notebook_node(nb) with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable: writable.write(ipynb)
def open_nb_and_strip_output(fname): cop = ClearOutputPreprocessor() with open(fname) as f: nb = nbformat.read(f, as_version=4) cop.preprocess(nb, dict()) return nb
def convert_notebook_to_assets(notebook_file_name, base_name, output_prefix): # define and create output folder output_folder = output_prefix + '/' + base_name os.makedirs(output_folder, exist_ok=True) # open file print('Converting Notebook: ' + notebook_file_name + ' ...') nb_file = open(notebook_file_name, 'r').read() nb = nbformat.reads(nb_file, as_version=4) # 1. clear output print(" - clearing output") ep = ClearOutputPreprocessor() ep.preprocess(nb, {}) # 2. generate fresh charts by running the notebook print(" - executing") ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=False) try: ep.preprocess(nb, {'metadata': {'path': output_folder}}) except Exception as e: print('ERROR: Execution of the notebook ' + notebook_file_name + ' stopped, likely for missing some py libs.') print( ' Please check the output/exception and add those to the requirements.' ) print(e) exit(1) # 3. export HTML print(" - generating html") cleaner_config = Config({ "HTMLExporter": { "exclude_input": True, "exclude_input_prompt": True, "exclude_output_prompt": True, "preprocessors": ['nbconvert.preprocessors.ExtractOutputPreprocessor'] }, }) local_templates = DictLoader({ 'our-html.tpl': stand_alone_tpl, 'react-glue.tpl': react_glue_tpl, }) exporter = HTMLExporter(config=cleaner_config, extra_loaders=[local_templates]) exporter.template_file = 'our-html.tpl' (html_body, html_resources) = exporter.from_notebook_node(nb) notebook_convert_time = current_utc_time() # save html output file, with local reference to the pictures local_html = [] output_html_file_name = output_folder + '/' + "index.html" print(" - saving html: " + output_html_file_name) with open(output_html_file_name, 'wt') as the_file: the_file.write(html_body) local_html.append({ 'notebook': base_name, 'notebook_html': output_html_file_name, 'convert_time': notebook_convert_time, }) # save js file for react inclusion (local ref to the pictures) # exporter.template_file = 'react-glue.tpl' # (react_body, react_resources) = exporter.from_notebook_node(nb) # output_react_file_name = output_folder + '/' + "index.js" # print(" - saving react js: " + output_react_file_name) # with open(output_react_file_name, 'wt') as the_file: # the_file.write(react_body) # save all the figures local_figures = [] figures = html_resources['outputs'] figures_count = len(figures) figure_index = 1 for figure_file in figures: output_figure_file_name = output_folder + '/' + figure_file print(" - saving png " + str(figure_index) + " of " + str(figures_count) + ": " + output_figure_file_name) if not figure_file.endswith('.png'): print("WARNING: figure is not a PNG file") continue with open(output_figure_file_name, 'wb') as the_file: the_file.write(figures[figure_file]) local_figures.append({ 'figure': figure_file, 'file': output_figure_file_name, 'notebook': base_name, 'notebook_html': output_html_file_name, 'convert_time': notebook_convert_time, }) # create an empty 'custom.css' custom_css_file_name = output_folder + '/' + 'custom.css' with open(custom_css_file_name, 'wt') as the_file: the_file.write("") # return a recap of all assets return local_html, local_figures
def test_func(self): cwd = os.getcwd() passing = True print("\n---------------------" " Testing {0}.ipynb " "---------------------".format(nbname)) if (nbname in self.ignore) or (nbname in self.py2_ignore and sys.version_info[0] == 2): print(" Skipping {}".format(nbname)) return run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1]) os.chdir(run_path) clear_output = ClearOutputPreprocessor() with open(nbpath) as nbfile: notebook = nbformat.read(nbfile, as_version=4) clear_output.preprocess(notebook, {}) execute = ExecutePreprocessor( timeout=timeout, kernel_name="python{}".format(sys.version_info[0]), allow_errors=True, ) out = execute.preprocess(notebook, {}) os.chdir(cwd) for cell in out[0]["cells"]: if "outputs" in cell.keys(): for output in cell["outputs"]: if output["output_type"] == "error": passing = False err_msg = [] for traceback in output["traceback"]: err_msg += ["{}".format(traceback)] err_msg = "\n".join(err_msg) msg = """ \n ... {} FAILED \n {} in cell [{}] \n-----------\n{}\n-----------\n """.format( nbname, output["ename"], cell["execution_count"], cell["source"], ) traceback = """ ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format(err_msg) print(u"{}".format(msg + traceback)) assert passing, msg print(" ... {0} Passed \n".format(nbname))
def test_func(self): cwd = os.getcwd() passing = True print("\n---------------------" " Testing {0}.ipynb " "---------------------".format(nbname)) if ((nbname in self.ignore) or (nbname in self.py2_ignore and sys.version_info[0] == 2)): print(" Skipping {}".format(nbname)) return run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1]) os.chdir(run_path) ep = ClearOutputPreprocessor( resources={'metadata': { 'path': run_path }}) with open(nbpath) as f: nb = nbformat.read(f, as_version=4) ep.preprocess(nb, {}) ex = ExecutePreprocessor( timeout=timeout, kernel_name='python{}'.format(sys.version_info[0]), allow_errors=True, resources={'metadata': { 'path': run_path }}) out = ex.preprocess(nb, {}) os.chdir(cwd) for cell in out[0]['cells']: if 'outputs' in cell.keys(): for output in cell['outputs']: if output['output_type'] == 'error': passing = False err_msg = [] for o in output['traceback']: err_msg += ["{}".format(o)] err_msg = "\n".join(err_msg) msg = """ \n ... {} FAILED \n {} in cell [{}] \n-----------\n{}\n-----------\n """.format( nbname, output['ename'], cell['execution_count'], cell['source'], ) traceback = """ ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format(err_msg) print(u"{}".format(msg + traceback)) assert passing, msg print(" ... {0} Passed \n".format(nbname))
def run_notebook(self, name, event=None, timeout=None): """ run a given notebook immediately. Args: name (str): the name of the jobfile event (str): an event name timeout (int): timeout in seconds Returns: Metadata of results See Also: * nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute kwargs # -- see ExecuteProcessor class # -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html ep_kwargs = { # avoid timeouts to stop kernel 'timeout': timeout, # avoid kernel at exit functions # -- this stops ipykernel AttributeError 'send_multipart' 'shutdown_kernel': 'immediate', # set kernel name, blank is default # -- e.g. python3, ir # -- see https://stackoverflow.com/a/47053020/890242 'kernel_name': '', } # other interesting options ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: resources = { 'metadata': { 'path': self.defaults.OMEGA_TMP, } } if not meta_job.kind_meta.get('keep_output', False): # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, resources) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' finally: del ep # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results