def clear_notebook(path): real_path = Path(path) body = "" with open(real_path, "r", encoding="utf-8") as nbfile: nb = nbformat.read(nbfile, as_version=4) orig_parameters = extract_parameters(nb) params = parameter_values(orig_parameters, SCOPETYPE="OPENADC", PLATFORM="CWLITEARM") new_nb = replace_definitions(nb, params, execute=False) co = ClearOutputPreprocessor() exporter = NotebookExporter() node, resources = co.preprocess(new_nb, {'metadata': {'path': './'}}) body, resources = exporter.from_notebook_node(node, resources) with open(real_path, "w", encoding="utf-8") as nbfile: nbfile.write(body)
def clear_notebooks(root): """Clear the outputs of documentation notebooks.""" # cleanup ignored files run(['git', 'clean', '-fdX', root]) # remove release/autograded/feedback if os.path.exists(os.path.join(root, "user_guide", "release")): shutil.rmtree(os.path.join(root, "user_guide", "release")) if os.path.exists(os.path.join(root, "user_guide", "autograded")): shutil.rmtree(os.path.join(root, "user_guide", "autograded")) if os.path.exists(os.path.join(root, "user_guide", "feedback")): shutil.rmtree(os.path.join(root, "user_guide", "feedback")) if os.path.exists(os.path.join(root, "user_guide", "downloaded", "ps1", "extracted")): shutil.rmtree(os.path.join(root, "user_guide", "downloaded", "ps1", "extracted")) print("Clearing outputs of notebooks in '{}'...".format(os.path.abspath(root))) preprocessor = ClearOutputPreprocessor() for dirpath, dirnames, filenames in os.walk(root): is_submitted = _check_if_directory_in_path(dirpath, 'submitted') for filename in sorted(filenames): if os.path.splitext(filename)[1] == '.ipynb': # read in the notebook pth = os.path.join(dirpath, filename) with open(pth, 'r') as fh: orig_nb = read(fh, 4) # copy the original notebook new_nb = deepcopy(orig_nb) # check outputs of all the cells if not is_submitted: new_nb = preprocessor.preprocess(new_nb, {})[0] # clear metadata new_nb.metadata = {} # write the notebook back to disk with open(pth, 'w') as fh: write(new_nb, fh, 4) if orig_nb != new_nb: print("Cleared '{}'".format(pth))
def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" km = self.kernel_manager.get_kernel(kernel_id) nb, resources = ClearOutputPreprocessor().preprocess( nb, {'metadata': { 'path': self.cwd }}) ep = VoilaExecutePreprocessor(config=self.traitlet_config) with ep.setup_preprocessor(nb, resources, km=km): for cell_idx, cell in enumerate(nb.cells): res = ep.preprocess_cell(cell, resources, cell_idx, store_history=False) yield res[0]
def clearNbOutput(): cwd = os.getcwd() # walk the test directory and find all notebooks for dirname, dirnames, filenames in os.walk(nbdir): if dirname != ".ipynb_checkpoints": for filename in filenames: if (filename.endswith(".ipynb") and not filename.endswith("-checkpoint.ipynb")): with open(os.path.join(dirname, filename)) as f: print("Clearing output from {}".format(filename)) nb = nbformat.read(f, as_version=4) ep = ClearOutputPreprocessor() ep.preprocess(nb, { "kernel_name": "python{}".format(sys.version_info[0]) }) print(" ... done\n") return True
def unexecute(self): """ Unexecutes the notebook i.e. removes all output cells """ _logger.info(f"Cleaning up notebook {self.nb} in {self.nb_dir}") if not self.executed_nb_path.exists(): _logger.warning( f"{self.executed_nb_path} not found, nothing to clean") return with open(self.executed_nb_path) as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) if nb['metadata'].get('docs_executed', None): nb['metadata'].pop('docs_executed') clear_executor = ClearOutputPreprocessor() clear_executor.preprocess(nb, {}) with open(self.executed_nb_path, 'w') as f: nbformat.write(nb, f)
async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, resources = ClearOutputPreprocessor().preprocess( nb, {'metadata': { 'path': self.cwd }}) stop_execution = False for cell_idx, cell in enumerate(nb.cells): if stop_execution: break try: res = await self.executor.execute_cell(cell, None, cell_idx, store_history=False) except TimeoutError: res = cell stop_execution = True yield res
def clear_notebook(path): """Clears output from a notebook Parameters ---------- path: str (optional) path to a Jupyter Notebook Returns ------- None """ with open(_get_path(path), encoding="utf-8") as f: nb = nbformat.read(f, as_version=4) pp = ClearOutputPreprocessor() pp.preprocess(nb, {}) with open(path, "w", encoding="utf-8") as f: nbformat.write(nb, f)
def clean(): _delete_generated_files() config = Config() config.NotebookExporter.preprocessors = [ClearOutputPreprocessor()] exporter = NotebookExporter(config=config) for notebook in get_files("ipynb"): with open(notebook, encoding="utf8") as nb_file: nb = nbformat.read(nb_file, as_version=4) if not nb.cells[-1]["source"]: nb.cells.pop() format_code_cells(nb) for cell_id, cell in enumerate(nb.cells): cell["id"] = f"{notebook.stem}-{cell_id}" ipynb, _ = exporter.from_notebook_node(nb) with open(f"{notebook.stem}.ipynb", "w", encoding="utf8") as writable: writable.write(ipynb)
def nb_exec(fname, name): ver = make_version(name) dirpath = os.path.join(data_dir, name, ver) execpath = os.path.join(dirpath, 'src') src_nb = os.path.join(execpath, 'src.ipynb') ensure_dir(dirpath) ensure_dir(execpath) logger.info(f'Copying {fname} to {src_nb}') shutil.copy2(fname, src_nb) with open(src_nb) as f: nb = nbformat.read(f, as_version=4) co = ClearOutputPreprocessor() nb, _ = co.preprocess(nb, {}) ep = ExecutePreprocessor(timeout=-1, kernel_name='python3') try: ep.preprocess(nb, {'metadata': {'path': execpath}}) except CellExecutionError as e: print(e) with open(os.path.join(dirpath, 'error.log'), 'w') as f: f.write(str(e)) print('Stack trace saved') with open(os.path.join(dirpath, 'output.ipynb'), 'w', encoding='utf-8') as f: nbformat.write(nb, f) exp = HTMLExporter() body, res = exp.from_notebook_node(nb) with open(os.path.join(dirpath, 'output.html'), 'w', encoding='utf-8') as f: f.write(body)
async def _jinja_cell_generator(self, nb, kernel_id): """Generator that will execute a single notebook cell at a time""" nb, resources = ClearOutputPreprocessor().preprocess( nb, {'metadata': { 'path': self.cwd }}) for cell_idx, input_cell in enumerate(nb.cells): try: task = asyncio.ensure_future( self.executor.execute_cell(input_cell, None, cell_idx, store_history=False)) while True: done, pending = await asyncio.wait( {task}, timeout=self.voila_configuration. http_keep_alive_timeout) if pending: # If not done within the timeout, we send a heartbeat # this is fundamentally to avoid browser/proxy read-timeouts, but # can be used in a template to give feedback to a user self.write("<script>voila_heartbeat()</script>\n") self.flush() continue output_cell = await task break except TimeoutError: output_cell = input_cell break except CellExecutionError: if self.executor.should_strip_error(): strip_code_cell_warnings(input_cell) self.executor.strip_code_cell_errors(input_cell) output_cell = input_cell break except Exception as e: self.log.exception('Error at server while executing cell: %r', input_cell) output_cell = nbformat.v4.new_code_cell() if self.executor.should_strip_error(): output_cell.outputs = [{ "output_type": "stream", "name": "stderr", "text": "An exception occurred at the server (not the notebook). {}" .format(self.executor.cell_error_instruction), }] else: output_cell.outputs = [{ 'output_type': 'error', 'ename': type(e).__name__, 'evalue': str(e), 'traceback': traceback.format_exception(*sys.exc_info()), }] finally: yield output_cell
def open_nb_and_strip_output(fname): cop = ClearOutputPreprocessor() with open(fname) as f: nb = nbformat.read(f, as_version=4) cop.preprocess(nb, dict()) return nb
def convert_notebook_to_assets(notebook_file_name, base_name, output_prefix): # define and create output folder output_folder = output_prefix + '/' + base_name os.makedirs(output_folder, exist_ok=True) # open file print('Converting Notebook: ' + notebook_file_name + ' ...') nb_file = open(notebook_file_name, 'r').read() nb = nbformat.reads(nb_file, as_version=4) # 1. clear output print(" - clearing output") ep = ClearOutputPreprocessor() ep.preprocess(nb, {}) # 2. generate fresh charts by running the notebook print(" - executing") ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=False) try: ep.preprocess(nb, {'metadata': {'path': output_folder}}) except Exception as e: print('ERROR: Execution of the notebook ' + notebook_file_name + ' stopped, likely for missing some py libs.') print( ' Please check the output/exception and add those to the requirements.' ) print(e) exit(1) # 3. export HTML print(" - generating html") cleaner_config = Config({ "HTMLExporter": { "exclude_input": True, "exclude_input_prompt": True, "exclude_output_prompt": True, "preprocessors": ['nbconvert.preprocessors.ExtractOutputPreprocessor'] }, }) local_templates = DictLoader({ 'our-html.tpl': stand_alone_tpl, 'react-glue.tpl': react_glue_tpl, }) exporter = HTMLExporter(config=cleaner_config, extra_loaders=[local_templates]) exporter.template_file = 'our-html.tpl' (html_body, html_resources) = exporter.from_notebook_node(nb) notebook_convert_time = current_utc_time() # save html output file, with local reference to the pictures local_html = [] output_html_file_name = output_folder + '/' + "index.html" print(" - saving html: " + output_html_file_name) with open(output_html_file_name, 'wt') as the_file: the_file.write(html_body) local_html.append({ 'notebook': base_name, 'notebook_html': output_html_file_name, 'convert_time': notebook_convert_time, }) # save js file for react inclusion (local ref to the pictures) # exporter.template_file = 'react-glue.tpl' # (react_body, react_resources) = exporter.from_notebook_node(nb) # output_react_file_name = output_folder + '/' + "index.js" # print(" - saving react js: " + output_react_file_name) # with open(output_react_file_name, 'wt') as the_file: # the_file.write(react_body) # save all the figures local_figures = [] figures = html_resources['outputs'] figures_count = len(figures) figure_index = 1 for figure_file in figures: output_figure_file_name = output_folder + '/' + figure_file print(" - saving png " + str(figure_index) + " of " + str(figures_count) + ": " + output_figure_file_name) if not figure_file.endswith('.png'): print("WARNING: figure is not a PNG file") continue with open(output_figure_file_name, 'wb') as the_file: the_file.write(figures[figure_file]) local_figures.append({ 'figure': figure_file, 'file': output_figure_file_name, 'notebook': base_name, 'notebook_html': output_html_file_name, 'convert_time': notebook_convert_time, }) # create an empty 'custom.css' custom_css_file_name = output_folder + '/' + 'custom.css' with open(custom_css_file_name, 'wt') as the_file: the_file.write("") # return a recap of all assets return local_html, local_figures
import os import glob from nbformat import read, write, current_nbformat from nbconvert.preprocessors import ClearOutputPreprocessor notebooks = glob.glob("*/*.ipynb") for nbpath in notebooks: print(nbpath) with open(nbpath, "r") as fh: nb = read(fh, current_nbformat) for cell in nb.cells: cell.metadata = {} nb, _ = ClearOutputPreprocessor().preprocess(nb, {}) nb.metadata = { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" } } with open(nbpath, "w") as fh: write(nb, fh, current_nbformat)
def test_func(self): cwd = os.getcwd() passing = True print("\n---------------------" " Testing {0}.ipynb " "---------------------".format(nbname)) if (nbname in self.ignore) or (nbname in self.py2_ignore and sys.version_info[0] == 2): print(" Skipping {}".format(nbname)) return run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1]) os.chdir(run_path) clear_output = ClearOutputPreprocessor() with open(nbpath) as nbfile: notebook = nbformat.read(nbfile, as_version=4) clear_output.preprocess(notebook, {}) execute = ExecutePreprocessor( timeout=timeout, kernel_name="python{}".format(sys.version_info[0]), allow_errors=True, ) out = execute.preprocess(notebook, {}) os.chdir(cwd) for cell in out[0]["cells"]: if "outputs" in cell.keys(): for output in cell["outputs"]: if output["output_type"] == "error": passing = False err_msg = [] for traceback in output["traceback"]: err_msg += ["{}".format(traceback)] err_msg = "\n".join(err_msg) msg = """ \n ... {} FAILED \n {} in cell [{}] \n-----------\n{}\n-----------\n """.format( nbname, output["ename"], cell["execution_count"], cell["source"], ) traceback = """ ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format(err_msg) print(u"{}".format(msg + traceback)) assert passing, msg print(" ... {0} Passed \n".format(nbname))
def test_func(self): cwd = os.getcwd() passing = True print("\n---------------------" " Testing {0}.ipynb " "---------------------".format(nbname)) if ((nbname in self.ignore) or (nbname in self.py2_ignore and sys.version_info[0] == 2)): print(" Skipping {}".format(nbname)) return run_path = os.path.sep.join(nbpath.split(os.path.sep)[:-1]) os.chdir(run_path) ep = ClearOutputPreprocessor( resources={'metadata': { 'path': run_path }}) with open(nbpath) as f: nb = nbformat.read(f, as_version=4) ep.preprocess(nb, {}) ex = ExecutePreprocessor( timeout=timeout, kernel_name='python{}'.format(sys.version_info[0]), allow_errors=True, resources={'metadata': { 'path': run_path }}) out = ex.preprocess(nb, {}) os.chdir(cwd) for cell in out[0]['cells']: if 'outputs' in cell.keys(): for output in cell['outputs']: if output['output_type'] == 'error': passing = False err_msg = [] for o in output['traceback']: err_msg += ["{}".format(o)] err_msg = "\n".join(err_msg) msg = """ \n ... {} FAILED \n {} in cell [{}] \n-----------\n{}\n-----------\n """.format( nbname, output['ename'], cell['execution_count'], cell['source'], ) traceback = """ ----------------- >> begin Traceback << ----------------- \n {}\n \n----------------- >> end Traceback << -----------------\n """.format(err_msg) print(u"{}".format(msg + traceback)) assert passing, msg print(" ... {0} Passed \n".format(nbname))
def execute(self, force=False): """ Executes the specified notebook file, and writes the executed notebook to a new file. Parameters ---------- force : bool, optional To force rerun notebook even if it has already been executed Returns ------- executed_nb_path : str, ``None`` The path to the executed notebook path, or ``None`` if ``write=False``. status: bool Whether the notebook executed without errors or not, 0 = ran without error, 1 = error """ with open(self.nb_path, encoding='utf-8') as f: nb = nbformat.read(f, as_version=IPYTHON_VERSION) is_executed = nb['metadata'].get('docs_executed') if is_executed == 'executed' and not force: _logger.warning( f"Notebook {self.nb} in {self.nb_dir} already executed, skipping," f"to force execute, parse argument -f") status = 0 else: # Execute the notebook _logger.info(f"Executing notebook {self.nb} in {self.nb_dir}") t0 = time.time() clear_executor = ClearOutputPreprocessor() executor = ExecutePreprocessor(**self.execute_kwargs) # First clean up the notebook and remove any cells that have been run clear_executor.preprocess(nb, {}) try: executor.preprocess(nb, {'metadata': {'path': self.nb_dir}}) execute_dict = {'docs_executed': 'executed'} nb['metadata'].update(execute_dict) status = 0 except CellExecutionError as err: execute_dict = {'docs_executed': 'errored'} nb['metadata'].update(execute_dict) _logger.error(f"Error executing notebook {self.nb}") _logger.error(err) status = 1 _logger.info(f"Finished running notebook ({time.time() - t0})") _logger.info( f"Writing executed notebook to {self.executed_nb_path}") # Makes sure original notebook isn't left blank in case of error during writing if self.overwrite: with open(self.temp_nb_path, 'w', encoding='utf-8') as f: nbformat.write(nb, f) shutil.copyfile(self.temp_nb_path, self.executed_nb_path) os.remove(self.temp_nb_path) else: with open(self.executed_nb_path, 'w', encoding='utf-8') as f: nbformat.write(nb, f) return self.executed_nb_path, status
def clear_output(self, nb): from nbconvert.preprocessors import ClearOutputPreprocessor return ClearOutputPreprocessor().preprocess(nb, {})
def run_notebook(self, name, event=None, timeout=None): """ run a given notebook immediately. Args: name (str): the name of the jobfile event (str): an event name timeout (int): timeout in seconds Returns: Metadata of results See Also: * nbconvert https://nbconvert.readthedocs.io/en/latest/execute_api.html """ notebook = self.get(name) meta_job = self.metadata(name) ts = datetime.datetime.now() # execute kwargs # -- see ExecuteProcessor class # -- see https://nbconvert.readthedocs.io/en/latest/execute_api.html ep_kwargs = { # avoid timeouts to stop kernel 'timeout': timeout, # avoid kernel at exit functions # -- this stops ipykernel AttributeError 'send_multipart' 'shutdown_kernel': 'immediate', # set kernel name, blank is default # -- e.g. python3, ir # -- see https://stackoverflow.com/a/47053020/890242 'kernel_name': '', } # other interesting options ep_kwargs.update(meta_job.kind_meta.get('ep_kwargs', {})) try: resources = { 'metadata': { 'path': self.defaults.OMEGA_TMP, } } if not meta_job.kind_meta.get('keep_output', False): # https://nbconvert.readthedocs.io/en/latest/api/preprocessors.html cp = ClearOutputPreprocessor() cp.preprocess(notebook, resources) ep = ExecutePreprocessor(**ep_kwargs) ep.preprocess(notebook, resources) except Exception as e: status = 'ERROR' message = str(e) else: status = 'OK' message = '' finally: del ep # record results meta_results = self.put(notebook, 'results/{name}_{ts}'.format(**locals())) meta_results.attributes['source_job'] = name meta_results.save() job_results = meta_job.attributes.get('job_results', []) job_results.append(meta_results.name) meta_job.attributes['job_results'] = job_results # record final job status job_runs = meta_job.attributes.get('job_runs', []) runstate = { 'status': status, 'ts': ts, 'message': message, 'results': meta_results.name if status == 'OK' else None } job_runs.append(runstate) meta_job.attributes['job_runs'] = job_runs # set event run state if event was specified if event: attrs = meta_job.attributes triggers = attrs['triggers'] = attrs.get('triggers', []) scheduled = (trigger for trigger in triggers if trigger['event-kind'] == 'scheduled') for trigger in scheduled: if event == trigger['event']: trigger['status'] = status trigger['ts'] = ts meta_job.save() return meta_results