def runtest(self):
"""
Run test is called by pytest for each of these nodes that are
collected i.e. a notebook cell. Runs all the cell tests in one
kernel without restarting. It is very common for ipython
notebooks to run through assuming a single kernel. The cells
are tested that they execute without errors and that the
output matches the output stored in the notebook.
"""
# Simply skip cell if configured to
if self.options['skip']:
pytest.skip()
kernel = self.parent.kernel
if not kernel.is_alive():
raise RuntimeError("Kernel dead on test start")
# Execute the code in the current cell in the kernel. Returns the
# message id of the corresponding response from iopub.
msg_id = kernel.execute_cell_input(
self.cell.source, allow_stdin=False)
# Timeout for the cell execution
# after code is sent for execution, the kernel sends a message on
# the shell channel. Timeout if no message received.
timeout = self.config.option.nbval_cell_timeout
timed_out_this_run = False
# Poll the shell channel to get a message
try:
self.parent.kernel.await_reply(msg_id, timeout=timeout)
except Empty: # Timeout reached
# Try to interrupt kernel, as this will give us traceback:
kernel.interrupt()
self.parent.timed_out = True
timed_out_this_run = True
# This list stores the output information for the entire cell
outs = []
# TODO: Only store if comparing with nbdime, to save on memory usage
self.test_outputs = outs
# Now get the outputs from the iopub channel
while True:
# The iopub channel broadcasts a range of messages. We keep reading
# them until we find the message containing the side-effects of our
# code execution.
try:
# Get a message from the kernel iopub channel
msg = self.parent.get_kernel_message(timeout=self.output_timeout)
except Empty:
# This is not working: ! The code will not be checked
# if the time is out (when the cell stops to be executed?)
# Halt kernel here!
kernel.stop()
if timed_out_this_run:
self.raise_cell_error(
"Timeout of %g seconds exceeded while executing cell."
" Failed to interrupt kernel in %d seconds, so "
"failing without traceback." %
(timeout, self.output_timeout),
)
else:
self.parent.timed_out = True
self.raise_cell_error(
"Timeout of %d seconds exceeded waiting for output." %
self.output_timeout,
)
# now we must handle the message by checking the type and reply
# info and we store the output of the cell in a notebook node object
msg_type = msg['msg_type']
reply = msg['content']
out = NotebookNode(output_type=msg_type)
# Is the iopub message related to this cell execution?
if msg['parent_header'].get('msg_id') != msg_id:
continue
# When the kernel starts to execute code, it will enter the 'busy'
# state and when it finishes, it will enter the 'idle' state.
# The kernel will publish state 'starting' exactly
# once at process startup.
if msg_type == 'status':
if reply['execution_state'] == 'idle':
break
else:
continue
# execute_input: To let all frontends know what code is
# being executed at any given time, these messages contain a
# re-broadcast of the code portion of an execute_request,
# along with the execution_count.
elif msg_type == 'execute_input':
continue
# com? execute reply?
elif msg_type.startswith('comm'):
continue
elif msg_type == 'execute_reply':
continue
# This message type is used to clear the output that is
# visible on the frontend
# elif msg_type == 'clear_output':
# outs = []
# continue
# elif (msg_type == 'clear_output'
# and msg_type['execution_state'] == 'idle'):
# outs = []
# continue
# 'execute_result' is equivalent to a display_data message.
# The object being displayed is passed to the display
# hook, i.e. the *result* of the execution.
# The only difference is that 'execute_result' has an
# 'execution_count' number which does not seems useful
# (we will filter it in the sanitize function)
#
# When the reply is display_data or execute_result,
# the dictionary contains
# a 'data' sub-dictionary with the 'text' AND the 'image/png'
# picture (in hexadecimal). There is also a 'metadata' entry
# but currently is not of much use, sometimes there is information
# as height and width of the image (CHECK the documentation)
# Thus we iterate through the keys (mimes) 'data' sub-dictionary
# to obtain the 'text' and 'image/png' information
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = reply['metadata']
out['data'] = reply['data']
outs.append(out)
if msg_type == 'execute_result':
out.execution_count = reply['execution_count']
# if the message is a stream then we store the output
elif msg_type == 'stream':
out.name = reply['name']
out.text = reply['text']
outs.append(out)
# if the message type is an error then an error has occurred during
# cell execution. Therefore raise a cell error and pass the
# traceback information.
elif msg_type == 'error':
# Store error in output first
out['ename'] = reply['ename']
out['evalue'] = reply['evalue']
out['traceback'] = reply['traceback']
outs.append(out)
if not self.options['check_exception']:
# Ensure we flush iopub before raising error
try:
self.parent.kernel.await_idle(msg_id, self.output_timeout)
except Empty:
self.stop()
raise RuntimeError('Timed out waiting for idle kernel!')
traceback = '\n' + '\n'.join(reply['traceback'])
if out['ename'] == 'KeyboardInterrupt' and self.parent.timed_out:
msg = "Timeout of %g seconds exceeded executing cell" % timeout
else:
msg = "Cell execution caused an exception"
self.raise_cell_error(msg, traceback)
# any other message type is not expected
# should this raise an error?
else:
print("unhandled iopub msg:", msg_type)
outs[:] = coalesce_streams(outs)
# Cells where the reference is not run, will not check outputs:
unrun = self.cell.execution_count is None
if unrun and self.cell.outputs:
self.raise_cell_error('Unrun reference cell has outputs')
# Compare if the outputs have the same number of lines
# and throw an error if it fails
# if len(outs) != len(self.cell.outputs):
# self.diff_number_outputs(outs, self.cell.outputs)
# failed = True
failed = False
if self.options['check'] and not unrun:
if not self.compare_outputs(outs, coalesce_streams(self.cell.outputs)):
failed = True
# If the comparison failed then we raise an exception.
if failed:
# The traceback containing the difference in the outputs is
# stored in the variable comparison_traceback
self.raise_cell_error(
"Cell outputs differ",
# Here we must put the traceback output:
'\n'.join(self.comparison_traceback),
)
def run(self, cell, timeout=None):
"""
Run a notebook cell in the IPythonKernel
Parameters
----------
cell : IPython.notebook.Cell
the cell to be run
timeout : int or None (default)
the time in seconds after which a cell is stopped and assumed to
have timed out. If set to None the value in `default_timeout`
is used
Returns
-------
list of outs
a list of NotebookNodes of the returned types. This is
similar to the list of outputs generated when a cell is run
"""
use_timeout = self.default_timeout
if timeout is not None:
use_timeout = timeout
if hasattr(cell, 'input'):
self.kc.execute(cell.input)
elif hasattr(cell, 'source'):
self.kc.execute(cell.source)
else:
raise AttributeError('No source/input key')
self.shell.get_msg(timeout=use_timeout)
outs = []
while True:
try:
msg = self.iopub.get_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.name = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
if hasattr(content, 'execution_count'):
out['execution_count'] = content['execution_count']
else:
out['execution_count'] = None
out['data'] = content['data']
out['metadata'] = content['metadata']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type.startswith('comm_'):
# widget updates and communication,
# which we will ignore and hope that it is not more serious
pass
else:
print "unhandled iopub msg:", msg_type, content
outs.append(out)
return outs
def runtest(self):
"""
Run test is called by pytest for each of these nodes that are
collected i.e. a notebook cell. Runs all the cell tests in one
kernel without restarting. It is very common for ipython
notebooks to run through assuming a single kernel. The cells
are tested that they execute without errors and that the
output matches the output stored in the notebook.
"""
# Simply skip cell if configured to
if self.options['skip']:
pytest.skip()
kernel = self.parent.kernel
if not kernel.is_alive():
raise RuntimeError("Kernel dead on test start")
# Execute the code in the current cell in the kernel. Returns the
# message id of the corresponding response from iopub.
msg_id = kernel.execute_cell_input(
self.cell.source, allow_stdin=False)
# Timeout for the cell execution
# after code is sent for execution, the kernel sends a message on
# the shell channel. Timeout if no message received.
timeout = self.config.option.nbval_cell_timeout
timed_out_this_run = False
# Poll the shell channel to get a message
try:
self.parent.kernel.await_reply(msg_id, timeout=timeout)
except Empty: # Timeout reached
# Try to interrupt kernel, as this will give us traceback:
kernel.interrupt()
self.parent.timed_out = True
timed_out_this_run = True
# This list stores the output information for the entire cell
outs = []
# TODO: Only store if comparing with nbdime, to save on memory usage
self.test_outputs = outs
# Now get the outputs from the iopub channel
while True:
# The iopub channel broadcasts a range of messages. We keep reading
# them until we find the message containing the side-effects of our
# code execution.
try:
# Get a message from the kernel iopub channel
msg = self.parent.get_kernel_message(timeout=self.output_timeout)
except Empty:
# This is not working: ! The code will not be checked
# if the time is out (when the cell stops to be executed?)
# Halt kernel here!
kernel.stop()
if timed_out_this_run:
self.raise_cell_error(
"Timeout of %g seconds exceeded while executing cell."
" Failed to interrupt kernel in %d seconds, so "
"failing without traceback." %
(timeout, self.output_timeout),
)
else:
self.parent.timed_out = True
self.raise_cell_error(
"Timeout of %d seconds exceeded waiting for output." %
self.output_timeout,
)
# now we must handle the message by checking the type and reply
# info and we store the output of the cell in a notebook node object
msg_type = msg['msg_type']
reply = msg['content']
out = NotebookNode(output_type=msg_type)
# Is the iopub message related to this cell execution?
if msg['parent_header'].get('msg_id') != msg_id:
continue
# When the kernel starts to execute code, it will enter the 'busy'
# state and when it finishes, it will enter the 'idle' state.
# The kernel will publish state 'starting' exactly
# once at process startup.
if msg_type == 'status':
if reply['execution_state'] == 'idle':
break
else:
continue
# execute_input: To let all frontends know what code is
# being executed at any given time, these messages contain a
# re-broadcast of the code portion of an execute_request,
# along with the execution_count.
elif msg_type == 'execute_input':
continue
# com? execute reply?
elif msg_type.startswith('comm'):
continue
elif msg_type == 'execute_reply':
continue
# This message type is used to clear the output that is
# visible on the frontend
# elif msg_type == 'clear_output':
# outs = []
# continue
# elif (msg_type == 'clear_output'
# and msg_type['execution_state'] == 'idle'):
# outs = []
# continue
# 'execute_result' is equivalent to a display_data message.
# The object being displayed is passed to the display
# hook, i.e. the *result* of the execution.
# The only difference is that 'execute_result' has an
# 'execution_count' number which does not seems useful
# (we will filter it in the sanitize function)
#
# When the reply is display_data or execute_result,
# the dictionary contains
# a 'data' sub-dictionary with the 'text' AND the 'image/png'
# picture (in hexadecimal). There is also a 'metadata' entry
# but currently is not of much use, sometimes there is information
# as height and width of the image (CHECK the documentation)
# Thus we iterate through the keys (mimes) 'data' sub-dictionary
# to obtain the 'text' and 'image/png' information
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = reply['metadata']
out['data'] = reply['data']
outs.append(out)
if msg_type == 'execute_result':
out.execution_count = reply['execution_count']
# if the message is a stream then we store the output
elif msg_type == 'stream':
out.name = reply['name']
out.text = reply['text']
outs.append(out)
# if the message type is an error then an error has occurred during
# cell execution. Therefore raise a cell error and pass the
# traceback information.
elif msg_type == 'error':
# Store error in output first
out['ename'] = reply['ename']
out['evalue'] = reply['evalue']
out['traceback'] = reply['traceback']
outs.append(out)
if not self.options['check_exception']:
# Ensure we flush iopub before raising error
try:
self.parent.kernel.await_idle(msg_id, self.output_timeout)
except Empty:
self.stop()
raise RuntimeError('Timed out waiting for idle kernel!')
traceback = '\n' + '\n'.join(reply['traceback'])
if out['ename'] == 'KeyboardInterrupt' and self.parent.timed_out:
msg = "Timeout of %g seconds exceeded executing cell" % timeout
else:
msg = "Cell execution caused an exception"
self.raise_cell_error(msg, traceback)
# any other message type is not expected
# should this raise an error?
else:
print("unhandled iopub msg:", msg_type)
outs[:] = coalesce_streams(outs)
# Cells where the reference is not run, will not check outputs:
#unrun = self.cell.execution_count is None
#if unrun and self.cell.outputs:
#self.raise_cell_error('Unrun reference cell has outputs')
# Compare if the outputs have the same number of lines
# and throw an error if it fails
# if len(outs) != len(self.cell.outputs):
# self.diff_number_outputs(outs, self.cell.outputs)
# failed = True
failed = False
if self.options['check'] and not unrun:
if not self.compare_outputs(outs, coalesce_streams(self.cell.outputs)):
failed = True
# If the comparison failed then we raise an exception.
if failed:
# The traceback containing the difference in the outputs is
# stored in the variable comparison_traceback
self.raise_cell_error(
"Cell outputs differ",
# Here we must put the traceback output:
'\n'.join(self.comparison_traceback),
)
def run_cell(self, index_cell, cell, clean_function=None):
'''
Runs a notebook cell and update the output of that cell inplace.
@param index_cell index of the cell
@param cell cell to execute
@param clean_function cleaning function to apply to the code before running it
@return output of the cell
'''
if self.detailed_log:
self.detailed_log(
"[run_cell] index_cell={0} clean_function={1}".format(
index_cell, clean_function))
iscell, codei = NotebookRunner.get_cell_code(cell)
self.fLOG('-- running cell:\n%s\n' % codei)
if self.detailed_log:
self.detailed_log(
'[run_cell] code=\n {0}'.format(
"\n ".join(codei.split("\n"))))
code = self.clean_code(codei)
if clean_function is not None:
code = clean_function(code)
if self.detailed_log:
self.detailed_log(
' cleaned code=\n {0}'.format(
"\n ".join(code.split("\n"))))
if len(code) == 0:
return ""
if self.kc is None:
raise ValueError( # pragma: no cover
"No kernel was started, specify kernel=True when initializing the instance."
)
self.kc.execute(code)
reply = self.kc.get_shell_msg()
reason = None
try:
status = reply['content']['status']
except KeyError: # pragma: no cover
status = 'error'
reason = "no status key in reply['content']"
if status == 'error':
ansi_escape = re.compile(r'\x1b[^m]*m')
try:
tr = [
ansi_escape.sub('', _)
for _ in reply['content']['traceback']
]
except KeyError: # pragma: no cover
tr = ["No traceback, available keys in reply['content']"] + \
list(reply['content'])
traceback_text = '\n'.join(tr)
self.fLOG("[nberror]\n", traceback_text)
if self.detailed_log:
self.detailed_log('[run_cell] ERROR=\n {0}'.format(
"\n ".join(traceback_text.split("\n"))))
else:
traceback_text = ''
self.fLOG('-- cell returned')
outs = list()
nbissue = 0
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty: # pragma: no cover
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
status = "error"
reason = "exception Empty was raised"
nbissue += 1
if nbissue > 10:
# the notebook is empty
return ""
else:
continue
content = msg['content']
msg_type = msg['msg_type']
if self.detailed_log:
self.detailed_log(' msg_type={0}'.format(msg_type))
out = NotebookNode(output_type=msg_type, metadata=dict())
if 'execution_count' in content:
if iscell:
cell['execution_count'] = content['execution_count']
out.execution_count = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
if msg_type == 'stream':
out.name = content['name']
# in msgspec 5, this is name, text
# in msgspec 4, this is name, data
if 'text' in content:
out.text = content['text']
else:
out.data = content['data']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
out.data = content['data']
elif msg_type in ('pyerr', 'error'):
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
out.name = 'stderr'
elif msg_type == 'clear_output':
outs = list()
continue
elif msg_type in ('comm_open', 'comm_msg', 'comm_close'):
# widgets in a notebook
out.data = content["data"]
out.comm_id = content["comm_id"]
else:
dcontent = "\n".join("{0}={1}".format(k, v)
for k, v in sorted(content.items()))
raise NotImplementedError( # pragma: no cover
"Unhandled iopub message: '{0}'\n--CONTENT--\n{1}".format(
msg_type, dcontent))
outs.append(out)
if self.detailed_log:
self.detailed_log(' out={0}'.format(type(out)))
if hasattr(out, "data"):
self.detailed_log(' out={0}'.format(out.data))
if iscell:
cell['outputs'] = outs
raw = []
for _ in outs:
try:
t = _.data
except AttributeError:
continue
# see MIMEMAP to see the available output type
for k, v in t.items():
if k.startswith("text"):
raw.append(v)
sraw = "\n".join(raw)
self.fLOG(sraw)
if self.detailed_log:
self.detailed_log(' sraw=\n {0}'.format(
"\n ".join(sraw.split("\n"))))
def reply2string(reply):
sreply = []
for k, v in sorted(reply.items()):
if isinstance(v, dict):
temp = []
for _, __ in sorted(v.items()):
temp.append(" [{0}]={1}".format(_, str(__)))
v_ = "\n".join(temp)
sreply.append("reply['{0}']=dict\n{1}".format(k, v_))
else:
sreply.append("reply['{0}']={1}".format(k, str(v)))
sreply = "\n".join(sreply)
return sreply
if status == 'error':
sreply = reply2string(reply)
if len(code) < 5:
scode = [code]
else:
scode = ""
mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \
"\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}"
raise NotebookError(
mes.format(code, traceback_text, sraw, sreply, index_cell,
len(code), scode, self.comment, status, reason,
self._filename))
if self.detailed_log:
self.detailed_log('[run_cell] status={0}'.format(status))
return outs
