def _run_code_type(outs, runner, msg_type, content):
out = NotebookNode(output_type=msg_type)
if msg_type in ('status', 'pyin', 'execute_input'):
return outs
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = runner.MIME_MAP[mime]
except KeyError:
raise NotImplementedError('unhandled mime type: %s' % mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
return outs
else:
raise NotImplementedError('unhandled iopub message: %s' % msg_type)
outs.append(out)
return outs
def run_cell(kc, cell, tout):
retval = kc.execute(cell.input)
retval = kc.get_shell_msg(timeout=tout)
print "CONTENT_STATUS: %s" % retval['content']['status']
if retval['content']['status'] == 'error':
print "ENAME: "
pprint(retval['content']['ename'])
print "EVALUE:"
pprint(retval['content']['evalue'])
print "TRACEBACK:"
for i in retval['content']['traceback']:
print i
outs = []
while True:
try:
msg = kc.get_iopub_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'execute_input'):
continue
elif msg_type == 'execute_input':
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
# print msg_type, content
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.stream = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = content['metadata']
for mime, data in content['data'].iteritems():
attr = mime.split('/')[-1].lower()
# this gets most right, but fix svg+html, plain
attr = attr.replace('+xml', '').replace('plain', 'text')
setattr(out, attr, data)
if msg_type == 'execute_result':
out.prompt_number = content['execution_count']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
else:
print "unhandled iopub msg:", msg_type
# print "msg_type: %s" % msg_type
outs.append(out)
return retval['content']['status'], outs
def runtest(self):
kc = self.parent.runner.kc
cell = self.cell
if ("SKIPCI" in self.cell_description) and ("CI" in os.environ):
return
if self.parent.fixture_cell:
kc.execute(self.parent.fixture_cell.input, allow_stdin=False)
if self.cell_description.lower().startswith("fixture") or self.cell_description.lower().startswith("setup"):
self.parent.fixture_cell = self.cell
kc.execute(cell.input, allow_stdin=False)
# XXX: the way it's currently implemented there doesn't seem to be a
# point in handling a timeout situation here, since raising an exception
# on timeout breaks the rest of the tests. The correct way to do it is to
# send interrupt to the kernel, and then report timeout, so that the
# rest of the tests can continue.
reply = kc.get_shell_msg()
status = reply['content']['status']
traceback_text = ''
if status == 'error':
traceback_text = 'Cell raised uncaught exception: \n' + \
'\n'.join(reply['content']['traceback'])
# extract various outputs and streams
outs = list()
timeout = 20
while True:
try:
msg = kc.get_iopub_msg(timeout=timeout)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle
# before the queue becomes empty,
# if it doesn't, something bad has happened
raise
content = msg['content']
msg_type = msg['msg_type']
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content: out.text = content['text']
# execute_result == Out[] (_)
elif msg_type in ('display_data', 'execute_result'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
raise NotImplementedError(f'unhandled mime type: {mime}')
json_encode = (mime == "application/json")
data_out = data if not json_encode else json.dumps(data)
setattr(out, "data_type", attr)
setattr(out, "data", data_out)
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output': pass
# ignore
#outs = list()
#continue
else:
raise NotImplementedError(f'unhandled iopub message: {msg_type}')
outs.append(out)
#pprint(outs)
if status == 'error':
# Get all output streams, so that we can display them in the exception
pyout = []
stdout_data = ''
stderr_data = ''
for out in outs:
if out.output_type == 'execute_result':
# text, html, json (others are binary)
if out.data_type in ["text", "html", "json"]:
pyout.append(out.data)
else:
pyout.append(f"[{out.data_type} object]")
elif 'stream' in out:
if out.stream == 'stdout':
stdout_data = out.text
elif out.stream == 'stderr':
stderr_data = out.text
raise IPyNbException(self.cell_num, self.cell.input, "\n".join(pyout),
stdout_data, stderr_data, traceback_text)
def runtest(self):
"""
Run test is called by pytest for each of these nodes that are
collected i.e. a notebook cell. Runs all the cell tests in one
kernel without restarting. It is very common for ipython
notebooks to run through assuming a single kernel. The cells
are tested that they execute without errors and that the
output matches the output stored in the notebook.
"""
# Simply skip cell if configured to
if self.options['skip']:
pytest.skip()
kernel = self.parent.kernel
if not kernel.is_alive():
raise RuntimeError("Kernel dead on test start")
# Execute the code in the current cell in the kernel. Returns the
# message id of the corresponding response from iopub.
msg_id = kernel.execute_cell_input(
self.cell.source, allow_stdin=False)
# Timeout for the cell execution
# after code is sent for execution, the kernel sends a message on
# the shell channel. Timeout if no message received.
timeout = self.config.option.nbval_cell_timeout
timed_out_this_run = False
# Poll the shell channel to get a message
try:
self.parent.kernel.await_reply(msg_id, timeout=timeout)
except Empty: # Timeout reached
# Try to interrupt kernel, as this will give us traceback:
kernel.interrupt()
self.parent.timed_out = True
timed_out_this_run = True
# This list stores the output information for the entire cell
outs = []
# TODO: Only store if comparing with nbdime, to save on memory usage
self.test_outputs = outs
# Now get the outputs from the iopub channel
while True:
# The iopub channel broadcasts a range of messages. We keep reading
# them until we find the message containing the side-effects of our
# code execution.
try:
# Get a message from the kernel iopub channel
msg = self.parent.get_kernel_message(timeout=self.output_timeout)
except Empty:
# This is not working: ! The code will not be checked
# if the time is out (when the cell stops to be executed?)
# Halt kernel here!
kernel.stop()
if timed_out_this_run:
self.raise_cell_error(
"Timeout of %g seconds exceeded while executing cell."
" Failed to interrupt kernel in %d seconds, so "
"failing without traceback." %
(timeout, self.output_timeout),
)
else:
self.parent.timed_out = True
self.raise_cell_error(
"Timeout of %d seconds exceeded waiting for output." %
self.output_timeout,
)
# now we must handle the message by checking the type and reply
# info and we store the output of the cell in a notebook node object
msg_type = msg['msg_type']
reply = msg['content']
out = NotebookNode(output_type=msg_type)
# Is the iopub message related to this cell execution?
if msg['parent_header'].get('msg_id') != msg_id:
continue
# When the kernel starts to execute code, it will enter the 'busy'
# state and when it finishes, it will enter the 'idle' state.
# The kernel will publish state 'starting' exactly
# once at process startup.
if msg_type == 'status':
if reply['execution_state'] == 'idle':
break
else:
continue
# execute_input: To let all frontends know what code is
# being executed at any given time, these messages contain a
# re-broadcast of the code portion of an execute_request,
# along with the execution_count.
elif msg_type == 'execute_input':
continue
# com? execute reply?
elif msg_type.startswith('comm'):
continue
elif msg_type == 'execute_reply':
continue
# This message type is used to clear the output that is
# visible on the frontend
# elif msg_type == 'clear_output':
# outs = []
# continue
# elif (msg_type == 'clear_output'
# and msg_type['execution_state'] == 'idle'):
# outs = []
# continue
# 'execute_result' is equivalent to a display_data message.
# The object being displayed is passed to the display
# hook, i.e. the *result* of the execution.
# The only difference is that 'execute_result' has an
# 'execution_count' number which does not seems useful
# (we will filter it in the sanitize function)
#
# When the reply is display_data or execute_result,
# the dictionary contains
# a 'data' sub-dictionary with the 'text' AND the 'image/png'
# picture (in hexadecimal). There is also a 'metadata' entry
# but currently is not of much use, sometimes there is information
# as height and width of the image (CHECK the documentation)
# Thus we iterate through the keys (mimes) 'data' sub-dictionary
# to obtain the 'text' and 'image/png' information
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = reply['metadata']
out['data'] = reply['data']
outs.append(out)
if msg_type == 'execute_result':
out.execution_count = reply['execution_count']
# if the message is a stream then we store the output
elif msg_type == 'stream':
out.name = reply['name']
out.text = reply['text']
outs.append(out)
# if the message type is an error then an error has occurred during
# cell execution. Therefore raise a cell error and pass the
# traceback information.
elif msg_type == 'error':
# Store error in output first
out['ename'] = reply['ename']
out['evalue'] = reply['evalue']
out['traceback'] = reply['traceback']
outs.append(out)
if not self.options['check_exception']:
# Ensure we flush iopub before raising error
try:
self.parent.kernel.await_idle(msg_id, self.output_timeout)
except Empty:
self.stop()
raise RuntimeError('Timed out waiting for idle kernel!')
traceback = '\n' + '\n'.join(reply['traceback'])
if out['ename'] == 'KeyboardInterrupt' and self.parent.timed_out:
msg = "Timeout of %g seconds exceeded executing cell" % timeout
else:
msg = "Cell execution caused an exception"
self.raise_cell_error(msg, traceback)
# any other message type is not expected
# should this raise an error?
else:
print("unhandled iopub msg:", msg_type)
outs[:] = coalesce_streams(outs)
# Cells where the reference is not run, will not check outputs:
unrun = self.cell.execution_count is None
if unrun and self.cell.outputs:
self.raise_cell_error('Unrun reference cell has outputs')
# Compare if the outputs have the same number of lines
# and throw an error if it fails
# if len(outs) != len(self.cell.outputs):
# self.diff_number_outputs(outs, self.cell.outputs)
# failed = True
failed = False
if self.options['check'] and not unrun:
if not self.compare_outputs(outs, coalesce_streams(self.cell.outputs)):
failed = True
# If the comparison failed then we raise an exception.
if failed:
# The traceback containing the difference in the outputs is
# stored in the variable comparison_traceback
self.raise_cell_error(
"Cell outputs differ",
# Here we must put the traceback output:
'\n'.join(self.comparison_traceback),
)
def runtest(self):
"""
Run test is called by pytest for each of these nodes that are
collected i.e. a notebook cell. Runs all the cell tests in one
kernel without restarting. It is very common for ipython
notebooks to run through assuming a single kernel. The cells
are tested that they execute without errors and that the
output matches the output stored in the notebook.
"""
# Simply skip cell if configured to
if self.options['skip']:
pytest.skip()
kernel = self.parent.kernel
if not kernel.is_alive():
raise RuntimeError("Kernel dead on test start")
# Execute the code in the current cell in the kernel. Returns the
# message id of the corresponding response from iopub.
msg_id = kernel.execute_cell_input(
self.cell.source, allow_stdin=False)
# Timeout for the cell execution
# after code is sent for execution, the kernel sends a message on
# the shell channel. Timeout if no message received.
timeout = self.config.option.nbval_cell_timeout
timed_out_this_run = False
# Poll the shell channel to get a message
try:
self.parent.kernel.await_reply(msg_id, timeout=timeout)
except Empty: # Timeout reached
# Try to interrupt kernel, as this will give us traceback:
kernel.interrupt()
self.parent.timed_out = True
timed_out_this_run = True
# This list stores the output information for the entire cell
outs = []
# TODO: Only store if comparing with nbdime, to save on memory usage
self.test_outputs = outs
# Now get the outputs from the iopub channel
while True:
# The iopub channel broadcasts a range of messages. We keep reading
# them until we find the message containing the side-effects of our
# code execution.
try:
# Get a message from the kernel iopub channel
msg = self.parent.get_kernel_message(timeout=self.output_timeout)
except Empty:
# This is not working: ! The code will not be checked
# if the time is out (when the cell stops to be executed?)
# Halt kernel here!
kernel.stop()
if timed_out_this_run:
self.raise_cell_error(
"Timeout of %g seconds exceeded while executing cell."
" Failed to interrupt kernel in %d seconds, so "
"failing without traceback." %
(timeout, self.output_timeout),
)
else:
self.parent.timed_out = True
self.raise_cell_error(
"Timeout of %d seconds exceeded waiting for output." %
self.output_timeout,
)
# now we must handle the message by checking the type and reply
# info and we store the output of the cell in a notebook node object
msg_type = msg['msg_type']
reply = msg['content']
out = NotebookNode(output_type=msg_type)
# Is the iopub message related to this cell execution?
if msg['parent_header'].get('msg_id') != msg_id:
continue
# When the kernel starts to execute code, it will enter the 'busy'
# state and when it finishes, it will enter the 'idle' state.
# The kernel will publish state 'starting' exactly
# once at process startup.
if msg_type == 'status':
if reply['execution_state'] == 'idle':
break
else:
continue
# execute_input: To let all frontends know what code is
# being executed at any given time, these messages contain a
# re-broadcast of the code portion of an execute_request,
# along with the execution_count.
elif msg_type == 'execute_input':
continue
# com? execute reply?
elif msg_type.startswith('comm'):
continue
elif msg_type == 'execute_reply':
continue
# This message type is used to clear the output that is
# visible on the frontend
# elif msg_type == 'clear_output':
# outs = []
# continue
# elif (msg_type == 'clear_output'
# and msg_type['execution_state'] == 'idle'):
# outs = []
# continue
# 'execute_result' is equivalent to a display_data message.
# The object being displayed is passed to the display
# hook, i.e. the *result* of the execution.
# The only difference is that 'execute_result' has an
# 'execution_count' number which does not seems useful
# (we will filter it in the sanitize function)
#
# When the reply is display_data or execute_result,
# the dictionary contains
# a 'data' sub-dictionary with the 'text' AND the 'image/png'
# picture (in hexadecimal). There is also a 'metadata' entry
# but currently is not of much use, sometimes there is information
# as height and width of the image (CHECK the documentation)
# Thus we iterate through the keys (mimes) 'data' sub-dictionary
# to obtain the 'text' and 'image/png' information
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = reply['metadata']
out['data'] = reply['data']
outs.append(out)
if msg_type == 'execute_result':
out.execution_count = reply['execution_count']
# if the message is a stream then we store the output
elif msg_type == 'stream':
out.name = reply['name']
out.text = reply['text']
outs.append(out)
# if the message type is an error then an error has occurred during
# cell execution. Therefore raise a cell error and pass the
# traceback information.
elif msg_type == 'error':
# Store error in output first
out['ename'] = reply['ename']
out['evalue'] = reply['evalue']
out['traceback'] = reply['traceback']
outs.append(out)
if not self.options['check_exception']:
# Ensure we flush iopub before raising error
try:
self.parent.kernel.await_idle(msg_id, self.output_timeout)
except Empty:
self.stop()
raise RuntimeError('Timed out waiting for idle kernel!')
traceback = '\n' + '\n'.join(reply['traceback'])
if out['ename'] == 'KeyboardInterrupt' and self.parent.timed_out:
msg = "Timeout of %g seconds exceeded executing cell" % timeout
else:
msg = "Cell execution caused an exception"
self.raise_cell_error(msg, traceback)
# any other message type is not expected
# should this raise an error?
else:
print("unhandled iopub msg:", msg_type)
outs[:] = coalesce_streams(outs)
# Cells where the reference is not run, will not check outputs:
#unrun = self.cell.execution_count is None
#if unrun and self.cell.outputs:
#self.raise_cell_error('Unrun reference cell has outputs')
# Compare if the outputs have the same number of lines
# and throw an error if it fails
# if len(outs) != len(self.cell.outputs):
# self.diff_number_outputs(outs, self.cell.outputs)
# failed = True
failed = False
if self.options['check'] and not unrun:
if not self.compare_outputs(outs, coalesce_streams(self.cell.outputs)):
failed = True
# If the comparison failed then we raise an exception.
if failed:
# The traceback containing the difference in the outputs is
# stored in the variable comparison_traceback
self.raise_cell_error(
"Cell outputs differ",
# Here we must put the traceback output:
'\n'.join(self.comparison_traceback),
)
def _execute_cell(cell, shell, iopub, timeout=300):
"""
Execute an IPython Notebook Cell and return the cell output.
Parameters
----------
cell : IPython.nbformat.current.NotebookNode
The IPython Notebook cell to execute.
shell : IPython.kernel.blocking.channels.BlockingShellChannel
The shell channel which the cell is submitted to for execution.
iopub : IPython.kernel.blocking.channels.BlockingIOPubChannel
The iopub channel used to retrieve the result of the execution.
timeout : int
The number of seconds to wait for the execution to finish before giving
up.
Returns
-------
cell_outputs : list
The list of NotebookNodes holding the result of the execution.
"""
# Execute input
shell.execute(cell.input)
exe_result = shell.get_shell_msg(timeout=timeout)
if exe_result['content']['status'] == 'error':
raise RuntimeError('Failed to execute cell due to error: {!r}'.format(
str(exe_result['content']['evalue'])))
cell_outputs = list()
# Poll for iopub messages until no more messages are available
while True:
try:
msg = iopub.get_iopub_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input', 'execute_result'):
continue
content = msg['content']
node = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
node.stream = content['name']
if 'text' in content:
# v4 notebook format
node.text = content['text']
else:
# v3 notebook format
node.text = content['data']
bug_text = 'Using Anaconda Cloud api site https://api.anaconda.org'
if bug_text in node.text:
# Ignore conda (spam) messages/warnings
continue
elif msg_type in ('display_data', 'pyout'):
node['metadata'] = content['metadata']
for mime, data in list(content['data'].items()):
attr = mime.split('/')[-1].lower()
attr = attr.replace('+xml', '').replace('plain', 'text')
setattr(node, attr, data)
if msg_type == 'pyout':
node.prompt_number = content['execution_count']
elif msg_type == 'pyerr':
node.ename = content['ename']
node.evalue = content['evalue']
node.traceback = content['traceback']
else:
raise RuntimeError('Unhandled iopub message of type: {}'.format(
msg_type))
cell_outputs.append(node)
return cell_outputs
def run_cell(self, cell, cidx):
'''
Run a notebook cell and update the output of that cell in-place.
'''
logging.debug('running cell {}'.format(cidx))
# logging.debug(u'cell.input {}'.format(cell.input))
self.kc.execute(cell.source)
reply = self.kc.get_shell_msg()
status = reply['content']['status']
max_mem = system_memory_used()
logging.info(' memory used: {}'.format(sizeof_fmt(max_mem)))
if status == 'error':
traceback_text = 'Cell raised uncaught exception: \n' + \
'\n'.join(reply['content']['traceback'])
traceback_text = remove_ansicolor(traceback_text)
if 'NoDataFound' not in traceback_text:
logging.error(traceback_text)
else:
logging.debug('run_cell ok')
outs = list()
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue
# becomes empty,
# if it doesn't, something bad has happened
logging.error("empty exception")
raise
content = msg['content']
msg_type = msg['msg_type']
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format but
# uses error/execute_result in the message spec. This does the
# translation needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
#if 'execution_count' in content:
#cell['prompt_number'] = content['execution_count']
#out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
# print(out.text, end='')
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
logging.error("unhandled mime")
raise NotImplementedError('unhandled mime type: %s' %
mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
else:
logging.error("unhandled iopub")
raise NotImplementedError('unhandled iopub message: %s' %
msg_type)
outs.append(out)
# NOTE: Ver 4 format still have 'pyout', Why?
cell['outputs'] = upgrade_outputs(outs)
logging.debug("status: {}".format(status))
if status == 'error':
if 'NoDataFound' in traceback_text:
raise NoDataFound(traceback_text.split('\n')[-1])
else:
logging.debug(u"NotebookError raised")
raise NotebookError(traceback_text)
def run(self, cell, timeout=None):
"""
Run a notebook cell in the IPythonKernel
Parameters
----------
cell : IPython.notebook.Cell
the cell to be run
timeout : int or None (default)
the time in seconds after which a cell is stopped and assumed to
have timed out. If set to None the value in `default_timeout`
is used
Returns
-------
list of outs
a list of NotebookNodes of the returned types. This is
similar to the list of outputs generated when a cell is run
"""
use_timeout = self.default_timeout
if timeout is not None:
use_timeout = timeout
if hasattr(cell, 'input'):
self.kc.execute(cell.input)
elif hasattr(cell, 'source'):
self.kc.execute(cell.source)
else:
raise AttributeError('No source/input key')
self.shell.get_msg(timeout=use_timeout)
outs = []
while True:
try:
msg = self.iopub.get_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.name = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
if hasattr(content, 'execution_count'):
out['execution_count'] = content['execution_count']
else:
out['execution_count'] = None
out['data'] = content['data']
out['metadata'] = content['metadata']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type.startswith('comm_'):
# widget updates and communication,
# which we will ignore and hope that it is not more serious
pass
else:
print "unhandled iopub msg:", msg_type, content
outs.append(out)
return outs
def run_cell(self, index_cell, cell, clean_function=None):
'''
Runs a notebook cell and update the output of that cell inplace.
@param index_cell index of the cell
@param cell cell to execute
@param clean_function cleaning function to apply to the code before running it
@return output of the cell
'''
if self.detailed_log:
self.detailed_log(
"[run_cell] index_cell={0} clean_function={1}".format(
index_cell, clean_function))
iscell, codei = NotebookRunner.get_cell_code(cell)
self.fLOG('-- running cell:\n%s\n' % codei)
if self.detailed_log:
self.detailed_log(
'[run_cell] code=\n {0}'.format(
"\n ".join(codei.split("\n"))))
code = self.clean_code(codei)
if clean_function is not None:
code = clean_function(code)
if self.detailed_log:
self.detailed_log(
' cleaned code=\n {0}'.format(
"\n ".join(code.split("\n"))))
if len(code) == 0:
return ""
if self.kc is None:
raise ValueError( # pragma: no cover
"No kernel was started, specify kernel=True when initializing the instance."
)
self.kc.execute(code)
reply = self.kc.get_shell_msg()
reason = None
try:
status = reply['content']['status']
except KeyError: # pragma: no cover
status = 'error'
reason = "no status key in reply['content']"
if status == 'error':
ansi_escape = re.compile(r'\x1b[^m]*m')
try:
tr = [
ansi_escape.sub('', _)
for _ in reply['content']['traceback']
]
except KeyError: # pragma: no cover
tr = ["No traceback, available keys in reply['content']"] + \
list(reply['content'])
traceback_text = '\n'.join(tr)
self.fLOG("[nberror]\n", traceback_text)
if self.detailed_log:
self.detailed_log('[run_cell] ERROR=\n {0}'.format(
"\n ".join(traceback_text.split("\n"))))
else:
traceback_text = ''
self.fLOG('-- cell returned')
outs = list()
nbissue = 0
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty: # pragma: no cover
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
status = "error"
reason = "exception Empty was raised"
nbissue += 1
if nbissue > 10:
# the notebook is empty
return ""
else:
continue
content = msg['content']
msg_type = msg['msg_type']
if self.detailed_log:
self.detailed_log(' msg_type={0}'.format(msg_type))
out = NotebookNode(output_type=msg_type, metadata=dict())
if 'execution_count' in content:
if iscell:
cell['execution_count'] = content['execution_count']
out.execution_count = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
if msg_type == 'stream':
out.name = content['name']
# in msgspec 5, this is name, text
# in msgspec 4, this is name, data
if 'text' in content:
out.text = content['text']
else:
out.data = content['data']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
out.data = content['data']
elif msg_type in ('pyerr', 'error'):
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
out.name = 'stderr'
elif msg_type == 'clear_output':
outs = list()
continue
elif msg_type in ('comm_open', 'comm_msg', 'comm_close'):
# widgets in a notebook
out.data = content["data"]
out.comm_id = content["comm_id"]
else:
dcontent = "\n".join("{0}={1}".format(k, v)
for k, v in sorted(content.items()))
raise NotImplementedError( # pragma: no cover
"Unhandled iopub message: '{0}'\n--CONTENT--\n{1}".format(
msg_type, dcontent))
outs.append(out)
if self.detailed_log:
self.detailed_log(' out={0}'.format(type(out)))
if hasattr(out, "data"):
self.detailed_log(' out={0}'.format(out.data))
if iscell:
cell['outputs'] = outs
raw = []
for _ in outs:
try:
t = _.data
except AttributeError:
continue
# see MIMEMAP to see the available output type
for k, v in t.items():
if k.startswith("text"):
raw.append(v)
sraw = "\n".join(raw)
self.fLOG(sraw)
if self.detailed_log:
self.detailed_log(' sraw=\n {0}'.format(
"\n ".join(sraw.split("\n"))))
def reply2string(reply):
sreply = []
for k, v in sorted(reply.items()):
if isinstance(v, dict):
temp = []
for _, __ in sorted(v.items()):
temp.append(" [{0}]={1}".format(_, str(__)))
v_ = "\n".join(temp)
sreply.append("reply['{0}']=dict\n{1}".format(k, v_))
else:
sreply.append("reply['{0}']={1}".format(k, str(v)))
sreply = "\n".join(sreply)
return sreply
if status == 'error':
sreply = reply2string(reply)
if len(code) < 5:
scode = [code]
else:
scode = ""
mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \
"\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}"
raise NotebookError(
mes.format(code, traceback_text, sraw, sreply, index_cell,
len(code), scode, self.comment, status, reason,
self._filename))
if self.detailed_log:
self.detailed_log('[run_cell] status={0}'.format(status))
return outs
def run_cell(self, line, code):
self.kc.execute(code, allow_stdin=True)
reply = self.kc.get_shell_msg()
status = reply['content']['status']
outs = list()
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
print("empty ?!")
raise
content = msg['content']
msg_type = msg['msg_type']
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if msg_type == 'pyout':
print(content['data']['text/plain'])
continue
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type in ('comm_open', 'comm_msg', 'comm_close'):
# TODO handle this msg ?!?!?!
continue
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
print("unhandled mime")
raise NotImplementedError('unhandled mime type: %s' %
mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
else:
print("unhandled " + msg_type)
raise NotImplementedError('unhandled iopub message: %s' %
msg_type)
outs.append(out)
# NOTE: Ver 4 format still have 'pyout', Why?
# upgrade_outputs(outs)
print(str(outs))
print("status: {}".format(status))
def runtest(self):
"""
Run test is called by pytest for each of these nodes that are
collected i.e. a notebook cell. Runs all the cell tests in one
kernel without restarting. It is very common for ipython
notebooks to run through assuming a single kernel. The cells
are tested that they execute without errors and that the
output matches the output stored in the notebook.
"""
# Execute the code in the current cell in the kernel. Returns the
# message id of the corresponding response from iopub.
msg_id = self.parent.kernel.execute_cell_input(
self.cell.source, allow_stdin=False)
# Timeout for the cell execution
# after code is sent for execution, the kernel sends a message on
# the shell channel. Timeout if no message received.
timeout = 2000
# Poll the shell channel to get a message
while True:
try:
msg = self.parent.get_kernel_message(stream='shell',
timeout=timeout)
except Empty:
raise NbCellError("Timeout of %d seconds exceeded"
" executing cell: %s" (timeout,
self.cell.input))
# Is this the message we are waiting for?
if msg['parent_header'].get('msg_id') == msg_id:
break
else:
continue
# This list stores the output information for the entire cell
outs = []
# TODO: Only store if comparing with nbdime, to save on memory usage
self.test_outputs = outs
# Now get the outputs from the iopub channel, need smaller timeout
timeout = 5
while True:
# The iopub channel broadcasts a range of messages. We keep reading
# them until we find the message containing the side-effects of our
# code execution.
try:
# Get a message from the kernel iopub channel
msg = self.parent.get_kernel_message(timeout=timeout)
except Empty:
# This is not working: ! The code will not be checked
# if the time is out (when the cell stops to be executed?)
raise NbCellError("Timeout of %d seconds exceeded"
" waiting for output.")
# now we must handle the message by checking the type and reply
# info and we store the output of the cell in a notebook node object
msg_type = msg['msg_type']
reply = msg['content']
out = NotebookNode(output_type=msg_type)
# Is the iopub message related to this cell execution?
if msg['parent_header'].get('msg_id') != msg_id:
continue
# When the kernel starts to execute code, it will enter the 'busy'
# state and when it finishes, it will enter the 'idle' state.
# The kernel will publish state 'starting' exactly
# once at process startup.
if msg_type == 'status':
if reply['execution_state'] == 'idle':
break
else:
continue
# execute_input: To let all frontends know what code is
# being executed at any given time, these messages contain a
# re-broadcast of the code portion of an execute_request,
# along with the execution_count.
elif msg_type == 'execute_input':
continue
# com? execute reply?
elif msg_type.startswith('comm'):
continue
elif msg_type == 'execute_reply':
continue
# This message type is used to clear the output that is
# visible on the frontend
# elif msg_type == 'clear_output':
# outs = []
# continue
# elif (msg_type == 'clear_output'
# and msg_type['execution_state'] == 'idle'):
# outs = []
# continue
# 'execute_result' is equivalent to a display_data message.
# The object being displayed is passed to the display
# hook, i.e. the *result* of the execution.
# The only difference is that 'execute_result' has an
# 'execution_count' number which does not seems useful
# (we will filter it in the sanitize function)
#
# When the reply is display_data or execute_count,
# the dictionary contains
# a 'data' sub-dictionary with the 'text' AND the 'image/png'
# picture (in hexadecimal). There is also a 'metadata' entry
# but currently is not of much use, sometimes there is information
# as height and width of the image (CHECK the documentation)
# Thus we iterate through the keys (mimes) 'data' sub-dictionary
# to obtain the 'text' and 'image/png' information
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = reply['metadata']
out['data'] = {}
for mime, data in six.iteritems(reply['data']):
# This could be useful for reference or backward compatibility
# attr = mime.split('/')[-1].lower()
# attr = attr.replace('+xml', '').replace('plain', 'text')
# setattr(out, attr, data)
# Return the relevant entries from data:
# plain/text, image/png, execution_count, etc
# We could use a mime types list for this (MAYBE)
out.data[mime] = data
outs.append(out)
if msg_type == 'execute_result':
out.execution_count = reply['execution_count']
# if the message is a stream then we store the output
elif msg_type == 'stream':
out.stream = reply['name']
out.text = reply['text']
outs.append(out)
# if the message type is an error then an error has occurred during
# cell execution. Therefore raise a cell error and pass the
# traceback information.
elif msg_type == 'error':
# Store error in output first
out['ename'] = reply['ename']
out['evalue'] = reply['evalue']
out['traceback'] = reply['traceback']
outs.append(out)
traceback = '\n' + '\n'.join(reply['traceback'])
raise NbCellError(self.cell_num, "Cell execution caused an exception",
self.cell.source, traceback)
# any other message type is not expected
# should this raise an error?
else:
print("unhandled iopub msg:", msg_type)
# Compare if the outputs have the same number of lines
# and throw an error if it fails
# if len(outs) != len(self.cell.outputs):
# self.diff_number_outputs(outs, self.cell.outputs)
# failed = True
failed = False
if self.docompare:
if not self.compare_outputs(outs, self.cell.outputs):
failed = True
# If the comparison failed then we raise an exception.
if failed:
# The traceback containing the difference in the outputs is
# stored in the variable comparison_traceback
raise NbCellError(self.cell_num,
"Cell outputs differ",
self.cell.source,
# Here we must put the traceback output:
'\n'.join(self.comparison_traceback))
def run_cell(self, index_cell, cell, clean_function=None):
'''
Run a notebook cell and update the output of that cell in-place.
@param index_cell index of the cell
@param cell cell to execute
@param clean_function cleaning function to apply to the code before running it
@return output of the cell
'''
iscell, codei = NotebookRunner.get_cell_code(cell)
self.fLOG('-- running cell:\n%s\n' % codei)
code = self.clean_code(codei)
if clean_function is not None:
code = clean_function(code)
if len(code) == 0:
return ""
if self.kc is None:
raise ValueError(
"No kernel was started, specify kernel=True when initializing the instance.")
self.kc.execute(code)
reply = self.kc.get_shell_msg()
reason = None
try:
status = reply['content']['status']
except KeyError:
status = 'error'
reason = "no status key in reply['content']"
if status == 'error':
ansi_escape = re.compile(r'\x1b[^m]*m')
try:
tr = [ansi_escape.sub('', _)
for _ in reply['content']['traceback']]
except KeyError:
tr = ["No traceback, available keys in reply['content']"] + \
[_ for _ in reply['content']]
traceback_text = '\n'.join(tr)
self.fLOG("ERR:\n", traceback_text)
else:
traceback_text = ''
self.fLOG('-- cell returned')
outs = list()
nbissue = 0
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
status = "error"
reason = "exception Empty was raised"
nbissue += 1
if nbissue > 10:
# the notebook is empty
return ""
else:
continue
content = msg['content']
msg_type = msg['msg_type']
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format but uses
# error/execute_result in the message spec. This does the translation
# needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
if iscell:
cell['prompt_number'] = content['execution_count']
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
# in msgspec 5, this is name, text
# in msgspec 4, this is name, data
if 'text' in content:
out.text = content['text']
else:
out.data = content['data']
elif msg_type in ('display_data', 'pyout'):
out.data = content['data']
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
elif msg_type == 'comm_open' or msg_type == 'comm_msg':
# widgets in a notebook
out.data = content["data"]
out.comm_id = content["comm_id"]
else:
dcontent = "\n".join("{0}={1}".format(k, v)
for k, v in sorted(content.items()))
raise NotImplementedError(
'unhandled iopub message: %s' % msg_type + "\nCONTENT:\n" + dcontent)
outs.append(out)
if iscell:
cell['outputs'] = outs
raw = []
for _ in outs:
try:
t = _.data
except AttributeError:
continue
# see MIMEMAP to see the available output type
for k, v in t.items():
if k.startswith("text"):
raw.append(v)
sraw = "\n".join(raw)
self.fLOG(sraw)
def reply2string(reply):
sreply = []
for k, v in sorted(reply.items()):
if isinstance(v, dict):
temp = []
for _, __ in sorted(v.items()):
temp.append(" [{0}]={1}".format(_, str(__)))
v = "\n".join(temp)
sreply.append("reply['{0}']=dict\n{1}".format(k, v))
else:
sreply.append("reply['{0}']={1}".format(k, str(v)))
sreply = "\n".join(sreply)
return sreply
if status == 'error':
sreply = reply2string(reply)
if len(code) < 5:
scode = [code]
else:
scode = ""
mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \
"\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}"
raise NotebookError(mes.format(
code, traceback_text, sraw, sreply, index_cell, len(
code), scode, self.comment, status, reason,
self._filename))
return outs