def preprocess_cell(self, cell, resources, index, **kwargs):
"""
Override if you want to apply some preprocessing to each cell.
Must return modified cell and resource dictionary.
Parameters
----------
cell : NotebookNode cell
Notebook cell being processed
resources : dictionary
Additional resources used in the conversion process. Allows
preprocessors to pass variables into the Jinja engine.
index : int
Index of the cell being processed
"""
self._check_assign_resources(resources)
# Because nbclient is an async library, we need to wrap the parent async call to generate a syncronous version.
cell = run_sync(NotebookClient.async_execute_cell)(self, cell, index, store_history=self.store_history)
return cell, self.resources
def some_sync_function():
return run_sync(some_async_function)()
class SoSPapermillNotebookClient(PapermillNotebookClient):
def __init__(self, nb_man, km=None, raise_on_iopub_timeout=True, **kw):
super(SoSPapermillNotebookClient,
self).__init__(nb_man,
km=km,
raise_on_iopub_timeout=raise_on_iopub_timeout,
**kw)
self._filename = nb_man.nb.metadata.papermill['input_path']
self._params_kernel = 'SoS'
self._parameters = []
def _prepare_meta(self, cell):
if not hasattr(cell.metadata, 'kernel'):
cell.metadata['kernel'] = 'SoS'
if hasattr(cell.metadata, 'tags'):
if 'parameters' in cell.metadata.tags:
self._params_kernel = cell.metadata['kernel']
if 'injected-parameters' in cell.metadata.tags:
if self._params_kernel != 'SoS':
cell.source = f'%put {" ".join(self._parameters)} --to {self._params_kernel}\n' + cell.source
cell.metadata['collapsed'] = True
if 'jupyter' not in cell.metadata:
cell.metadata['jupyter'] = {}
cell.metadata['jupyter']['outputs_hidden'] = True
cell.metadata['jupyter']['source_hidden'] = True
meta = {
'use_panel': True,
'cell_id': randomCellID(),
'path': self._filename,
'batch_mode': True,
'cell_kernel': cell.metadata.kernel
}
if re.search(
r'^%sosrun($|\s)|^%sossave($|\s)|^%preview\s.*(-w|--workflow).*$',
cell.source, re.MULTILINE):
meta['workflow'] = self._workflow
return meta
async def async_execute_cell(self,
cell,
cell_index=0,
execution_count=None,
store_history=True):
# sos is the additional meta information sent to kernel
if cell.cell_type != 'code' or not cell.source.strip():
self.log.debug("Skipping non-executing cell %s", cell_index)
return cell
if self.record_timing and 'execution' not in cell['metadata']:
cell['metadata']['execution'] = {}
sos_meta = self._prepare_meta(cell)
content = dict(code=cell.source,
silent=False,
store_history=store_history,
user_expressions='',
allow_stdin=False,
stop_on_error=False,
sos=sos_meta)
msg = self.kc.session.msg('execute_request', content)
self.kc.shell_channel.send(msg)
msg_id = msg['header']['msg_id']
# the reset is copied from https://github.com/jupyter/nbconvert/blob/master/nbconvert/preprocessors/execute.py
# because we only need to change the first line
# msg_id = self.kc.execute(cell.source)
self.log.debug(
f"Executing cell {cell_index} with kernel {content['sos']['cell_kernel']}:\n{cell.source}"
)
exec_reply = await self.async_wait_for_reply(msg_id, cell)
self.code_cells_executed += 1
exec_timeout = self._get_timeout(cell)
cell.outputs = []
self.clear_before_next_output = False
while True:
try:
# We've already waited for execute_reply, so all output
# should already be waiting. However, on slow networks, like
# in certain CI systems, waiting < 1 second might miss messages.
# So long as the kernel sends a status:idle message when it
# finishes, we won't actually have to wait this long, anyway.
msg = await ensure_async(
self.kc.iopub_channel.get_msg(timeout=self.iopub_timeout))
except Empty:
self.log.warning("Timeout waiting for IOPub output")
if self.raise_on_iopub_timeout:
raise RuntimeError("Timeout waiting for IOPub output")
else:
break
if msg['parent_header'].get('msg_id') != msg_id:
# not an output from our execution
continue
msg_type = msg['msg_type']
self.log.debug("output: %s", msg_type)
content = msg['content']
# set the prompt number for the input and the output
if 'execution_count' in exec_reply['content']:
cell['execution_count'] = exec_reply['content'][
'execution_count']
if msg_type == 'status':
if content['execution_state'] == 'idle':
break
else:
continue
elif msg_type == 'execute_input':
continue
elif msg_type == 'clear_output':
cell.outputs[:] = []
# clear display_id mapping for this cell
for display_id, cell_map in self._display_id_map.items():
if cell_index in cell_map:
cell_map[cell_index] = []
continue
elif msg_type.startswith('comm'):
continue
display_id = None
if msg_type in {
'execute_result', 'display_data', 'update_display_data'
}:
display_id = msg['content'].get('transient',
{}).get('display_id', None)
if display_id:
self._update_display_id(display_id, msg)
if msg_type == 'update_display_data':
# update_display_data doesn't get recorded
continue
try:
out = output_from_msg(msg)
except ValueError:
self.log.error("unhandled iopub msg: " + msg_type)
continue
if display_id:
# record output index in:
# _display_id_map[display_id][cell_idx]
cell_map = self._display_id_map.setdefault(display_id, {})
output_idx_list = cell_map.setdefault(cell_index, [])
output_idx_list.append(len(cell.outputs))
cell.outputs.append(out)
return cell
execute_cell = run_sync(async_execute_cell)
def execute(self, **kwargs):
self._workflow = extract_workflow(self.nb_man.nb)
self._parameters = list(
self.nb_man.nb.metadata['papermill']['parameters'].keys())
return super(SoSPapermillNotebookClient, self).execute(**kwargs)
class NMAPreprocessor(ExecutePreprocessor):
"""
Custom subclass of the ExecutePreprocessor for NMA tutorials.
This class overwrites the execute_cell method to ignore NotImplementedError
exceptions, which are raised when incomplete exercise functions are called.
All other errors will be handled as normal.
"""
# Note: we have to patch the entire async_execute_cell method because it checks
# for errors with a private method (_check_raise_for_error). It would be cleaner
# to customize only the error handling method, but alas, that is not allowed.
# The nbconvert.ExecutePreprocessor class inherits from both
# nbconvert.Preprocessor and nbclient.NotebookClient. The relevant methods that we
# patch are defined on the latter. The code here was taken from this specific tag:
# https://github.com/jupyter/nbclient/blob/0.5.1/nbclient/client.py
async def async_execute_cell(
self,
cell: nbformat.NotebookNode,
cell_index: int,
execution_count: t.Optional[int] = None,
store_history: bool = True) -> nbformat.NotebookNode:
"""
Executes a single code cell.
To execute all cells see :meth:`execute`.
Parameters
----------
cell : nbformat.NotebookNode
The cell which is currently being processed.
cell_index : int
The position of the cell within the notebook object.
execution_count : int
The execution count assigned to the cell (default: Use kernel response)
store_history : bool
Determines if history should be stored in the kernel (default: False).
Specific to ipython kernels, which can store command histories.
Raises
------
CellExecutionError
If execution failed and should raise an exception, this will be raised
with defaults about the failure.
Returns
-------
cell : NotebookNode
The cell which was just processed.
License
-------
This project is licensed under the terms of the Modified BSD License
(also known as New or Revised or 3-Clause BSD), as follows:
- Copyright (c) 2020-, Jupyter Development Team
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice, this
list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.
Neither the name of the Jupyter Development Team nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
assert self.kc is not None
if cell.cell_type != 'code' or not cell.source.strip():
self.log.debug("Skipping non-executing cell %s", cell_index)
return cell
if self.record_timing and 'execution' not in cell['metadata']:
cell['metadata']['execution'] = {}
self.log.debug("Executing cell:\n%s", cell.source)
parent_msg_id = await ensure_async(
self.kc.execute(cell.source,
store_history=store_history,
stop_on_error=not self.allow_errors))
# We launched a code cell to execute
self.code_cells_executed += 1
exec_timeout = self._get_timeout(cell)
cell.outputs = []
self.clear_before_next_output = False
task_poll_kernel_alive = asyncio.ensure_future(
self._async_poll_kernel_alive())
task_poll_output_msg = asyncio.ensure_future(
self._async_poll_output_msg(parent_msg_id, cell, cell_index))
self.task_poll_for_reply = asyncio.ensure_future(
self._async_poll_for_reply(
parent_msg_id,
cell,
exec_timeout,
task_poll_output_msg,
task_poll_kernel_alive,
))
try:
exec_reply = await self.task_poll_for_reply
except asyncio.CancelledError:
# can only be cancelled by task_poll_kernel_alive when the kernel is dead
task_poll_output_msg.cancel()
raise DeadKernelError("Kernel died")
except Exception as e:
# Best effort to cancel request if it hasn't been resolved
try:
# Check if the task_poll_output is doing the raising for us
if not isinstance(e, CellControlSignal):
task_poll_output_msg.cancel()
finally:
raise
if execution_count:
cell['execution_count'] = execution_count
# -- NMA-specific code here -- #
self._check_raise_for_error_nma(cell, exec_reply)
self.nb['cells'][cell_index] = cell
return cell
def _check_raise_for_error_nma(self, cell: nbformat.NotebookNode,
exec_reply: t.Optional[t.Dict]) -> None:
cell_tags = cell.metadata.get("tags", [])
cell_allows_errors = self.allow_errors or "raises-exception" in cell_tags
if self.force_raise_errors or not cell_allows_errors:
if (exec_reply is not None
) and exec_reply['content']['status'] == 'error':
# -- NMA-specific code here -- #
if exec_reply['content']['ename'] != 'NotImplementedError':
raise CellExecutionError.from_cell_and_msg(
cell, exec_reply['content'])
execute_cell = run_sync(async_execute_cell)
class IllusionistClient(NotebookClient, Application):
"""
Extends NBClient to add some utilities to run commands in the kernel
"""
store_widget_state = True
async def async_execute(self, reset_kc=False, **kwargs):
"""
We overwrite and copy the original code for this function from
nbclient.NotebookClient and change a couple of things
We need to do this because output widgets are
not being updated if I run code in a function after even
after keeping the kernel alive (not sure why).
"""
if reset_kc and self.km:
await self._async_cleanup_kernel()
self.reset_execution_trackers()
async with self.async_setup_kernel(**kwargs):
self.log.info(
"Executing notebook with kernel: %s" % self.kernel_name
)
for index, cell in enumerate(self.nb.cells):
# Ignore `'execution_count' in content` as it's always 1
# when store_history is False
await self.async_execute_cell(
cell, index, execution_count=self.code_cells_executed + 1
)
# ADDED
await ensure_async(self.after_notebook())
# NED
msg_id = await ensure_async(self.kc.kernel_info())
info_msg = await self.async_wait_for_reply(msg_id)
self.nb.metadata["language_info"] = info_msg["content"][
"language_info"
]
self.set_widgets_metadata()
return self.nb
execute = run_sync(async_execute)
def after_notebook(self):
"""
This is called after the regular notebook cells have been executed
"""
async def async_exec_code(self, source, write_cell=settings.dev_mode):
"""
Execute code in a Kernel
Parameters
----------
source (str): Execute this code
write_cell (bool, default=None): Write a new cell to the notebook
Returns
-------
NotebookNode
"""
cell = nbformat.NotebookNode()
cell.cell_type = "code"
cell.execution_count = self.code_cells_executed + 1
cell.metadata = {}
cell.outputs = []
cell.source = source
self.nb["cells"].append(cell)
cell_index = len(self.nb["cells"]) - 1
cell = await self.async_execute_cell(
cell, cell_index, execution_count=cell.execution_count
)
if write_cell == False:
# Delete created cell
del self.nb["cells"][cell_index]
return cell
exec_code = run_sync(async_exec_code)
def eval_cell(self, cell):
"""
Run Python `eval` on a NotebookCell
Parameters
----------
cell (NotebookNode):
Returns
-------
Python objects
"""
output_txt = cell["outputs"][0]["data"]["text/plain"]
return eval(output_txt)