def preprocess_cell(self, cell, resources, index, **kwargs): """ Override if you want to apply some preprocessing to each cell. Must return modified cell and resource dictionary. Parameters ---------- cell : NotebookNode cell Notebook cell being processed resources : dictionary Additional resources used in the conversion process. Allows preprocessors to pass variables into the Jinja engine. index : int Index of the cell being processed """ self._check_assign_resources(resources) # Because nbclient is an async library, we need to wrap the parent async call to generate a syncronous version. cell = run_sync(NotebookClient.async_execute_cell)(self, cell, index, store_history=self.store_history) return cell, self.resources
def some_sync_function(): return run_sync(some_async_function)()
class SoSPapermillNotebookClient(PapermillNotebookClient): def __init__(self, nb_man, km=None, raise_on_iopub_timeout=True, **kw): super(SoSPapermillNotebookClient, self).__init__(nb_man, km=km, raise_on_iopub_timeout=raise_on_iopub_timeout, **kw) self._filename = nb_man.nb.metadata.papermill['input_path'] self._params_kernel = 'SoS' self._parameters = [] def _prepare_meta(self, cell): if not hasattr(cell.metadata, 'kernel'): cell.metadata['kernel'] = 'SoS' if hasattr(cell.metadata, 'tags'): if 'parameters' in cell.metadata.tags: self._params_kernel = cell.metadata['kernel'] if 'injected-parameters' in cell.metadata.tags: if self._params_kernel != 'SoS': cell.source = f'%put {" ".join(self._parameters)} --to {self._params_kernel}\n' + cell.source cell.metadata['collapsed'] = True if 'jupyter' not in cell.metadata: cell.metadata['jupyter'] = {} cell.metadata['jupyter']['outputs_hidden'] = True cell.metadata['jupyter']['source_hidden'] = True meta = { 'use_panel': True, 'cell_id': randomCellID(), 'path': self._filename, 'batch_mode': True, 'cell_kernel': cell.metadata.kernel } if re.search( r'^%sosrun($|\s)|^%sossave($|\s)|^%preview\s.*(-w|--workflow).*$', cell.source, re.MULTILINE): meta['workflow'] = self._workflow return meta async def async_execute_cell(self, cell, cell_index=0, execution_count=None, store_history=True): # sos is the additional meta information sent to kernel if cell.cell_type != 'code' or not cell.source.strip(): self.log.debug("Skipping non-executing cell %s", cell_index) return cell if self.record_timing and 'execution' not in cell['metadata']: cell['metadata']['execution'] = {} sos_meta = self._prepare_meta(cell) content = dict(code=cell.source, silent=False, store_history=store_history, user_expressions='', allow_stdin=False, stop_on_error=False, sos=sos_meta) msg = self.kc.session.msg('execute_request', content) self.kc.shell_channel.send(msg) msg_id = msg['header']['msg_id'] # the reset is copied from https://github.com/jupyter/nbconvert/blob/master/nbconvert/preprocessors/execute.py # because we only need to change the first line # msg_id = self.kc.execute(cell.source) self.log.debug( f"Executing cell {cell_index} with kernel {content['sos']['cell_kernel']}:\n{cell.source}" ) exec_reply = await self.async_wait_for_reply(msg_id, cell) self.code_cells_executed += 1 exec_timeout = self._get_timeout(cell) cell.outputs = [] self.clear_before_next_output = False while True: try: # We've already waited for execute_reply, so all output # should already be waiting. However, on slow networks, like # in certain CI systems, waiting < 1 second might miss messages. # So long as the kernel sends a status:idle message when it # finishes, we won't actually have to wait this long, anyway. msg = await ensure_async( self.kc.iopub_channel.get_msg(timeout=self.iopub_timeout)) except Empty: self.log.warning("Timeout waiting for IOPub output") if self.raise_on_iopub_timeout: raise RuntimeError("Timeout waiting for IOPub output") else: break if msg['parent_header'].get('msg_id') != msg_id: # not an output from our execution continue msg_type = msg['msg_type'] self.log.debug("output: %s", msg_type) content = msg['content'] # set the prompt number for the input and the output if 'execution_count' in exec_reply['content']: cell['execution_count'] = exec_reply['content'][ 'execution_count'] if msg_type == 'status': if content['execution_state'] == 'idle': break else: continue elif msg_type == 'execute_input': continue elif msg_type == 'clear_output': cell.outputs[:] = [] # clear display_id mapping for this cell for display_id, cell_map in self._display_id_map.items(): if cell_index in cell_map: cell_map[cell_index] = [] continue elif msg_type.startswith('comm'): continue display_id = None if msg_type in { 'execute_result', 'display_data', 'update_display_data' }: display_id = msg['content'].get('transient', {}).get('display_id', None) if display_id: self._update_display_id(display_id, msg) if msg_type == 'update_display_data': # update_display_data doesn't get recorded continue try: out = output_from_msg(msg) except ValueError: self.log.error("unhandled iopub msg: " + msg_type) continue if display_id: # record output index in: # _display_id_map[display_id][cell_idx] cell_map = self._display_id_map.setdefault(display_id, {}) output_idx_list = cell_map.setdefault(cell_index, []) output_idx_list.append(len(cell.outputs)) cell.outputs.append(out) return cell execute_cell = run_sync(async_execute_cell) def execute(self, **kwargs): self._workflow = extract_workflow(self.nb_man.nb) self._parameters = list( self.nb_man.nb.metadata['papermill']['parameters'].keys()) return super(SoSPapermillNotebookClient, self).execute(**kwargs)
class NMAPreprocessor(ExecutePreprocessor): """ Custom subclass of the ExecutePreprocessor for NMA tutorials. This class overwrites the execute_cell method to ignore NotImplementedError exceptions, which are raised when incomplete exercise functions are called. All other errors will be handled as normal. """ # Note: we have to patch the entire async_execute_cell method because it checks # for errors with a private method (_check_raise_for_error). It would be cleaner # to customize only the error handling method, but alas, that is not allowed. # The nbconvert.ExecutePreprocessor class inherits from both # nbconvert.Preprocessor and nbclient.NotebookClient. The relevant methods that we # patch are defined on the latter. The code here was taken from this specific tag: # https://github.com/jupyter/nbclient/blob/0.5.1/nbclient/client.py async def async_execute_cell( self, cell: nbformat.NotebookNode, cell_index: int, execution_count: t.Optional[int] = None, store_history: bool = True) -> nbformat.NotebookNode: """ Executes a single code cell. To execute all cells see :meth:`execute`. Parameters ---------- cell : nbformat.NotebookNode The cell which is currently being processed. cell_index : int The position of the cell within the notebook object. execution_count : int The execution count assigned to the cell (default: Use kernel response) store_history : bool Determines if history should be stored in the kernel (default: False). Specific to ipython kernels, which can store command histories. Raises ------ CellExecutionError If execution failed and should raise an exception, this will be raised with defaults about the failure. Returns ------- cell : NotebookNode The cell which was just processed. License ------- This project is licensed under the terms of the Modified BSD License (also known as New or Revised or 3-Clause BSD), as follows: - Copyright (c) 2020-, Jupyter Development Team All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of the Jupyter Development Team nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ assert self.kc is not None if cell.cell_type != 'code' or not cell.source.strip(): self.log.debug("Skipping non-executing cell %s", cell_index) return cell if self.record_timing and 'execution' not in cell['metadata']: cell['metadata']['execution'] = {} self.log.debug("Executing cell:\n%s", cell.source) parent_msg_id = await ensure_async( self.kc.execute(cell.source, store_history=store_history, stop_on_error=not self.allow_errors)) # We launched a code cell to execute self.code_cells_executed += 1 exec_timeout = self._get_timeout(cell) cell.outputs = [] self.clear_before_next_output = False task_poll_kernel_alive = asyncio.ensure_future( self._async_poll_kernel_alive()) task_poll_output_msg = asyncio.ensure_future( self._async_poll_output_msg(parent_msg_id, cell, cell_index)) self.task_poll_for_reply = asyncio.ensure_future( self._async_poll_for_reply( parent_msg_id, cell, exec_timeout, task_poll_output_msg, task_poll_kernel_alive, )) try: exec_reply = await self.task_poll_for_reply except asyncio.CancelledError: # can only be cancelled by task_poll_kernel_alive when the kernel is dead task_poll_output_msg.cancel() raise DeadKernelError("Kernel died") except Exception as e: # Best effort to cancel request if it hasn't been resolved try: # Check if the task_poll_output is doing the raising for us if not isinstance(e, CellControlSignal): task_poll_output_msg.cancel() finally: raise if execution_count: cell['execution_count'] = execution_count # -- NMA-specific code here -- # self._check_raise_for_error_nma(cell, exec_reply) self.nb['cells'][cell_index] = cell return cell def _check_raise_for_error_nma(self, cell: nbformat.NotebookNode, exec_reply: t.Optional[t.Dict]) -> None: cell_tags = cell.metadata.get("tags", []) cell_allows_errors = self.allow_errors or "raises-exception" in cell_tags if self.force_raise_errors or not cell_allows_errors: if (exec_reply is not None ) and exec_reply['content']['status'] == 'error': # -- NMA-specific code here -- # if exec_reply['content']['ename'] != 'NotImplementedError': raise CellExecutionError.from_cell_and_msg( cell, exec_reply['content']) execute_cell = run_sync(async_execute_cell)
class IllusionistClient(NotebookClient, Application): """ Extends NBClient to add some utilities to run commands in the kernel """ store_widget_state = True async def async_execute(self, reset_kc=False, **kwargs): """ We overwrite and copy the original code for this function from nbclient.NotebookClient and change a couple of things We need to do this because output widgets are not being updated if I run code in a function after even after keeping the kernel alive (not sure why). """ if reset_kc and self.km: await self._async_cleanup_kernel() self.reset_execution_trackers() async with self.async_setup_kernel(**kwargs): self.log.info( "Executing notebook with kernel: %s" % self.kernel_name ) for index, cell in enumerate(self.nb.cells): # Ignore `'execution_count' in content` as it's always 1 # when store_history is False await self.async_execute_cell( cell, index, execution_count=self.code_cells_executed + 1 ) # ADDED await ensure_async(self.after_notebook()) # NED msg_id = await ensure_async(self.kc.kernel_info()) info_msg = await self.async_wait_for_reply(msg_id) self.nb.metadata["language_info"] = info_msg["content"][ "language_info" ] self.set_widgets_metadata() return self.nb execute = run_sync(async_execute) def after_notebook(self): """ This is called after the regular notebook cells have been executed """ async def async_exec_code(self, source, write_cell=settings.dev_mode): """ Execute code in a Kernel Parameters ---------- source (str): Execute this code write_cell (bool, default=None): Write a new cell to the notebook Returns ------- NotebookNode """ cell = nbformat.NotebookNode() cell.cell_type = "code" cell.execution_count = self.code_cells_executed + 1 cell.metadata = {} cell.outputs = [] cell.source = source self.nb["cells"].append(cell) cell_index = len(self.nb["cells"]) - 1 cell = await self.async_execute_cell( cell, cell_index, execution_count=cell.execution_count ) if write_cell == False: # Delete created cell del self.nb["cells"][cell_index] return cell exec_code = run_sync(async_exec_code) def eval_cell(self, cell): """ Run Python `eval` on a NotebookCell Parameters ---------- cell (NotebookNode): Returns ------- Python objects """ output_txt = cell["outputs"][0]["data"]["text/plain"] return eval(output_txt)