def _run_code_type(outs, runner, msg_type, content):
out = NotebookNode(output_type=msg_type)
if msg_type in ('status', 'pyin', 'execute_input'):
return outs
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = runner.MIME_MAP[mime]
except KeyError:
raise NotImplementedError('unhandled mime type: %s' % mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
return outs
else:
raise NotImplementedError('unhandled iopub message: %s' % msg_type)
outs.append(out)
return outs
def run_cell(kc, cell, tout):
retval = kc.execute(cell.input)
retval = kc.get_shell_msg(timeout=tout)
print "CONTENT_STATUS: %s" % retval['content']['status']
if retval['content']['status'] == 'error':
print "ENAME: "
pprint(retval['content']['ename'])
print "EVALUE:"
pprint(retval['content']['evalue'])
print "TRACEBACK:"
for i in retval['content']['traceback']:
print i
outs = []
while True:
try:
msg = kc.get_iopub_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'execute_input'):
continue
elif msg_type == 'execute_input':
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
# print msg_type, content
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.stream = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'execute_result'):
out['metadata'] = content['metadata']
for mime, data in content['data'].iteritems():
attr = mime.split('/')[-1].lower()
# this gets most right, but fix svg+html, plain
attr = attr.replace('+xml', '').replace('plain', 'text')
setattr(out, attr, data)
if msg_type == 'execute_result':
out.prompt_number = content['execution_count']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
else:
print "unhandled iopub msg:", msg_type
# print "msg_type: %s" % msg_type
outs.append(out)
return retval['content']['status'], outs
def runtest(self):
kc = self.parent.runner.kc
cell = self.cell
if ("SKIPCI" in self.cell_description) and ("CI" in os.environ):
return
if self.parent.fixture_cell:
kc.execute(self.parent.fixture_cell.input, allow_stdin=False)
if self.cell_description.lower().startswith("fixture") or self.cell_description.lower().startswith("setup"):
self.parent.fixture_cell = self.cell
kc.execute(cell.input, allow_stdin=False)
# XXX: the way it's currently implemented there doesn't seem to be a
# point in handling a timeout situation here, since raising an exception
# on timeout breaks the rest of the tests. The correct way to do it is to
# send interrupt to the kernel, and then report timeout, so that the
# rest of the tests can continue.
reply = kc.get_shell_msg()
status = reply['content']['status']
traceback_text = ''
if status == 'error':
traceback_text = 'Cell raised uncaught exception: \n' + \
'\n'.join(reply['content']['traceback'])
# extract various outputs and streams
outs = list()
timeout = 20
while True:
try:
msg = kc.get_iopub_msg(timeout=timeout)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle
# before the queue becomes empty,
# if it doesn't, something bad has happened
raise
content = msg['content']
msg_type = msg['msg_type']
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content: out.text = content['text']
# execute_result == Out[] (_)
elif msg_type in ('display_data', 'execute_result'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
raise NotImplementedError(f'unhandled mime type: {mime}')
json_encode = (mime == "application/json")
data_out = data if not json_encode else json.dumps(data)
setattr(out, "data_type", attr)
setattr(out, "data", data_out)
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output': pass
# ignore
#outs = list()
#continue
else:
raise NotImplementedError(f'unhandled iopub message: {msg_type}')
outs.append(out)
#pprint(outs)
if status == 'error':
# Get all output streams, so that we can display them in the exception
pyout = []
stdout_data = ''
stderr_data = ''
for out in outs:
if out.output_type == 'execute_result':
# text, html, json (others are binary)
if out.data_type in ["text", "html", "json"]:
pyout.append(out.data)
else:
pyout.append(f"[{out.data_type} object]")
elif 'stream' in out:
if out.stream == 'stdout':
stdout_data = out.text
elif out.stream == 'stderr':
stderr_data = out.text
raise IPyNbException(self.cell_num, self.cell.input, "\n".join(pyout),
stdout_data, stderr_data, traceback_text)
def _execute_cell(cell, shell, iopub, timeout=300):
"""
Execute an IPython Notebook Cell and return the cell output.
Parameters
----------
cell : IPython.nbformat.current.NotebookNode
The IPython Notebook cell to execute.
shell : IPython.kernel.blocking.channels.BlockingShellChannel
The shell channel which the cell is submitted to for execution.
iopub : IPython.kernel.blocking.channels.BlockingIOPubChannel
The iopub channel used to retrieve the result of the execution.
timeout : int
The number of seconds to wait for the execution to finish before giving
up.
Returns
-------
cell_outputs : list
The list of NotebookNodes holding the result of the execution.
"""
# Execute input
shell.execute(cell.input)
exe_result = shell.get_shell_msg(timeout=timeout)
if exe_result['content']['status'] == 'error':
raise RuntimeError('Failed to execute cell due to error: {!r}'.format(
str(exe_result['content']['evalue'])))
cell_outputs = list()
# Poll for iopub messages until no more messages are available
while True:
try:
msg = iopub.get_iopub_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input', 'execute_result'):
continue
content = msg['content']
node = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
node.stream = content['name']
if 'text' in content:
# v4 notebook format
node.text = content['text']
else:
# v3 notebook format
node.text = content['data']
bug_text = 'Using Anaconda Cloud api site https://api.anaconda.org'
if bug_text in node.text:
# Ignore conda (spam) messages/warnings
continue
elif msg_type in ('display_data', 'pyout'):
node['metadata'] = content['metadata']
for mime, data in list(content['data'].items()):
attr = mime.split('/')[-1].lower()
attr = attr.replace('+xml', '').replace('plain', 'text')
setattr(node, attr, data)
if msg_type == 'pyout':
node.prompt_number = content['execution_count']
elif msg_type == 'pyerr':
node.ename = content['ename']
node.evalue = content['evalue']
node.traceback = content['traceback']
else:
raise RuntimeError('Unhandled iopub message of type: {}'.format(
msg_type))
cell_outputs.append(node)
return cell_outputs
def run_cell(self, cell, cidx):
'''
Run a notebook cell and update the output of that cell in-place.
'''
logging.debug('running cell {}'.format(cidx))
# logging.debug(u'cell.input {}'.format(cell.input))
self.kc.execute(cell.source)
reply = self.kc.get_shell_msg()
status = reply['content']['status']
max_mem = system_memory_used()
logging.info(' memory used: {}'.format(sizeof_fmt(max_mem)))
if status == 'error':
traceback_text = 'Cell raised uncaught exception: \n' + \
'\n'.join(reply['content']['traceback'])
traceback_text = remove_ansicolor(traceback_text)
if 'NoDataFound' not in traceback_text:
logging.error(traceback_text)
else:
logging.debug('run_cell ok')
outs = list()
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue
# becomes empty,
# if it doesn't, something bad has happened
logging.error("empty exception")
raise
content = msg['content']
msg_type = msg['msg_type']
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format but
# uses error/execute_result in the message spec. This does the
# translation needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
#if 'execution_count' in content:
#cell['prompt_number'] = content['execution_count']
#out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
# print(out.text, end='')
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
logging.error("unhandled mime")
raise NotImplementedError('unhandled mime type: %s' %
mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
else:
logging.error("unhandled iopub")
raise NotImplementedError('unhandled iopub message: %s' %
msg_type)
outs.append(out)
# NOTE: Ver 4 format still have 'pyout', Why?
cell['outputs'] = upgrade_outputs(outs)
logging.debug("status: {}".format(status))
if status == 'error':
if 'NoDataFound' in traceback_text:
raise NoDataFound(traceback_text.split('\n')[-1])
else:
logging.debug(u"NotebookError raised")
raise NotebookError(traceback_text)
def run(self, cell, timeout=None):
"""
Run a notebook cell in the IPythonKernel
Parameters
----------
cell : IPython.notebook.Cell
the cell to be run
timeout : int or None (default)
the time in seconds after which a cell is stopped and assumed to
have timed out. If set to None the value in `default_timeout`
is used
Returns
-------
list of outs
a list of NotebookNodes of the returned types. This is
similar to the list of outputs generated when a cell is run
"""
use_timeout = self.default_timeout
if timeout is not None:
use_timeout = timeout
if hasattr(cell, 'input'):
self.kc.execute(cell.input)
elif hasattr(cell, 'source'):
self.kc.execute(cell.source)
else:
raise AttributeError('No source/input key')
self.shell.get_msg(timeout=use_timeout)
outs = []
while True:
try:
msg = self.iopub.get_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.name = content['name']
out.text = content['text']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
if hasattr(content, 'execution_count'):
out['execution_count'] = content['execution_count']
else:
out['execution_count'] = None
out['data'] = content['data']
out['metadata'] = content['metadata']
elif msg_type == 'error':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type.startswith('comm_'):
# widget updates and communication,
# which we will ignore and hope that it is not more serious
pass
else:
print "unhandled iopub msg:", msg_type, content
outs.append(out)
return outs
def run_cell(self, index_cell, cell, clean_function=None):
'''
Runs a notebook cell and update the output of that cell inplace.
@param index_cell index of the cell
@param cell cell to execute
@param clean_function cleaning function to apply to the code before running it
@return output of the cell
'''
if self.detailed_log:
self.detailed_log(
"[run_cell] index_cell={0} clean_function={1}".format(
index_cell, clean_function))
iscell, codei = NotebookRunner.get_cell_code(cell)
self.fLOG('-- running cell:\n%s\n' % codei)
if self.detailed_log:
self.detailed_log(
'[run_cell] code=\n {0}'.format(
"\n ".join(codei.split("\n"))))
code = self.clean_code(codei)
if clean_function is not None:
code = clean_function(code)
if self.detailed_log:
self.detailed_log(
' cleaned code=\n {0}'.format(
"\n ".join(code.split("\n"))))
if len(code) == 0:
return ""
if self.kc is None:
raise ValueError( # pragma: no cover
"No kernel was started, specify kernel=True when initializing the instance."
)
self.kc.execute(code)
reply = self.kc.get_shell_msg()
reason = None
try:
status = reply['content']['status']
except KeyError: # pragma: no cover
status = 'error'
reason = "no status key in reply['content']"
if status == 'error':
ansi_escape = re.compile(r'\x1b[^m]*m')
try:
tr = [
ansi_escape.sub('', _)
for _ in reply['content']['traceback']
]
except KeyError: # pragma: no cover
tr = ["No traceback, available keys in reply['content']"] + \
list(reply['content'])
traceback_text = '\n'.join(tr)
self.fLOG("[nberror]\n", traceback_text)
if self.detailed_log:
self.detailed_log('[run_cell] ERROR=\n {0}'.format(
"\n ".join(traceback_text.split("\n"))))
else:
traceback_text = ''
self.fLOG('-- cell returned')
outs = list()
nbissue = 0
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty: # pragma: no cover
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
status = "error"
reason = "exception Empty was raised"
nbissue += 1
if nbissue > 10:
# the notebook is empty
return ""
else:
continue
content = msg['content']
msg_type = msg['msg_type']
if self.detailed_log:
self.detailed_log(' msg_type={0}'.format(msg_type))
out = NotebookNode(output_type=msg_type, metadata=dict())
if 'execution_count' in content:
if iscell:
cell['execution_count'] = content['execution_count']
out.execution_count = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
if msg_type == 'stream':
out.name = content['name']
# in msgspec 5, this is name, text
# in msgspec 4, this is name, data
if 'text' in content:
out.text = content['text']
else:
out.data = content['data']
elif msg_type in ('display_data', 'pyout', 'execute_result'):
out.data = content['data']
elif msg_type in ('pyerr', 'error'):
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
out.name = 'stderr'
elif msg_type == 'clear_output':
outs = list()
continue
elif msg_type in ('comm_open', 'comm_msg', 'comm_close'):
# widgets in a notebook
out.data = content["data"]
out.comm_id = content["comm_id"]
else:
dcontent = "\n".join("{0}={1}".format(k, v)
for k, v in sorted(content.items()))
raise NotImplementedError( # pragma: no cover
"Unhandled iopub message: '{0}'\n--CONTENT--\n{1}".format(
msg_type, dcontent))
outs.append(out)
if self.detailed_log:
self.detailed_log(' out={0}'.format(type(out)))
if hasattr(out, "data"):
self.detailed_log(' out={0}'.format(out.data))
if iscell:
cell['outputs'] = outs
raw = []
for _ in outs:
try:
t = _.data
except AttributeError:
continue
# see MIMEMAP to see the available output type
for k, v in t.items():
if k.startswith("text"):
raw.append(v)
sraw = "\n".join(raw)
self.fLOG(sraw)
if self.detailed_log:
self.detailed_log(' sraw=\n {0}'.format(
"\n ".join(sraw.split("\n"))))
def reply2string(reply):
sreply = []
for k, v in sorted(reply.items()):
if isinstance(v, dict):
temp = []
for _, __ in sorted(v.items()):
temp.append(" [{0}]={1}".format(_, str(__)))
v_ = "\n".join(temp)
sreply.append("reply['{0}']=dict\n{1}".format(k, v_))
else:
sreply.append("reply['{0}']={1}".format(k, str(v)))
sreply = "\n".join(sreply)
return sreply
if status == 'error':
sreply = reply2string(reply)
if len(code) < 5:
scode = [code]
else:
scode = ""
mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \
"\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}"
raise NotebookError(
mes.format(code, traceback_text, sraw, sreply, index_cell,
len(code), scode, self.comment, status, reason,
self._filename))
if self.detailed_log:
self.detailed_log('[run_cell] status={0}'.format(status))
return outs
def run_cell(self, line, code):
self.kc.execute(code, allow_stdin=True)
reply = self.kc.get_shell_msg()
status = reply['content']['status']
outs = list()
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
print("empty ?!")
raise
content = msg['content']
msg_type = msg['msg_type']
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if msg_type == 'pyout':
print(content['data']['text/plain'])
continue
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type in ('comm_open', 'comm_msg', 'comm_close'):
# TODO handle this msg ?!?!?!
continue
elif msg_type == 'stream':
out.stream = content['name']
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
print("unhandled mime")
raise NotImplementedError('unhandled mime type: %s' %
mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
else:
print("unhandled " + msg_type)
raise NotImplementedError('unhandled iopub message: %s' %
msg_type)
outs.append(out)
# NOTE: Ver 4 format still have 'pyout', Why?
# upgrade_outputs(outs)
print(str(outs))
print("status: {}".format(status))
def run_cell(self, index_cell, cell, clean_function=None):
'''
Run a notebook cell and update the output of that cell in-place.
@param index_cell index of the cell
@param cell cell to execute
@param clean_function cleaning function to apply to the code before running it
@return output of the cell
'''
iscell, codei = NotebookRunner.get_cell_code(cell)
self.fLOG('-- running cell:\n%s\n' % codei)
code = self.clean_code(codei)
if clean_function is not None:
code = clean_function(code)
if len(code) == 0:
return ""
if self.kc is None:
raise ValueError(
"No kernel was started, specify kernel=True when initializing the instance.")
self.kc.execute(code)
reply = self.kc.get_shell_msg()
reason = None
try:
status = reply['content']['status']
except KeyError:
status = 'error'
reason = "no status key in reply['content']"
if status == 'error':
ansi_escape = re.compile(r'\x1b[^m]*m')
try:
tr = [ansi_escape.sub('', _)
for _ in reply['content']['traceback']]
except KeyError:
tr = ["No traceback, available keys in reply['content']"] + \
[_ for _ in reply['content']]
traceback_text = '\n'.join(tr)
self.fLOG("ERR:\n", traceback_text)
else:
traceback_text = ''
self.fLOG('-- cell returned')
outs = list()
nbissue = 0
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
status = "error"
reason = "exception Empty was raised"
nbissue += 1
if nbissue > 10:
# the notebook is empty
return ""
else:
continue
content = msg['content']
msg_type = msg['msg_type']
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format but uses
# error/execute_result in the message spec. This does the translation
# needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
if iscell:
cell['prompt_number'] = content['execution_count']
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
# in msgspec 5, this is name, text
# in msgspec 4, this is name, data
if 'text' in content:
out.text = content['text']
else:
out.data = content['data']
elif msg_type in ('display_data', 'pyout'):
out.data = content['data']
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
elif msg_type == 'comm_open' or msg_type == 'comm_msg':
# widgets in a notebook
out.data = content["data"]
out.comm_id = content["comm_id"]
else:
dcontent = "\n".join("{0}={1}".format(k, v)
for k, v in sorted(content.items()))
raise NotImplementedError(
'unhandled iopub message: %s' % msg_type + "\nCONTENT:\n" + dcontent)
outs.append(out)
if iscell:
cell['outputs'] = outs
raw = []
for _ in outs:
try:
t = _.data
except AttributeError:
continue
# see MIMEMAP to see the available output type
for k, v in t.items():
if k.startswith("text"):
raw.append(v)
sraw = "\n".join(raw)
self.fLOG(sraw)
def reply2string(reply):
sreply = []
for k, v in sorted(reply.items()):
if isinstance(v, dict):
temp = []
for _, __ in sorted(v.items()):
temp.append(" [{0}]={1}".format(_, str(__)))
v = "\n".join(temp)
sreply.append("reply['{0}']=dict\n{1}".format(k, v))
else:
sreply.append("reply['{0}']={1}".format(k, str(v)))
sreply = "\n".join(sreply)
return sreply
if status == 'error':
sreply = reply2string(reply)
if len(code) < 5:
scode = [code]
else:
scode = ""
mes = "FILENAME\n{10}:1:1\n{7}\nCELL status={8}, reason={9} -- {4} length={5} -- {6}:\n-----------------\n{0}" + \
"\n-----------------\nTRACE:\n{1}\nRAW:\n{2}REPLY:\n{3}"
raise NotebookError(mes.format(
code, traceback_text, sraw, sreply, index_cell, len(
code), scode, self.comment, status, reason,
self._filename))
return outs
