def __init__(self,
notebook_dir,
extra_args=None,
profile=None,
timeout=90):
self.notebook_dir = os.path.abspath(notebook_dir)
self.profile = profile
self.timeout = timeout
km = KernelManager()
if extra_args is None:
extra_args = []
if profile is not None:
extra_args += ["--profile=%s" % profile]
km.start_kernel(stderr=open(os.devnull, 'w'),
extra_arguments=extra_args)
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError: # still on 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# make sure it's working
shell.execute("pass")
shell.get_msg()
# all of these should be run pylab inline
shell.execute("%pylab inline")
shell.get_msg()
self.kc = kc
self.km = km
self.iopub = iopub
def run_notebook(nb, cell_filter = lambda cell: cell,
extra_arguments=['--pylab=inline', '--profile=stats'],
modify_outputs=True,
run_cells=True,
timeout=10):
"""
Take a notebook and send all its cells to a kernel.
Takes an optional filter to modify the results of the
cell after being run and having its
output set by `run_cell` if modify_outputs is True.
"""
km = KernelManager()
km.start_kernel(extra_arguments=extra_arguments,
stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
successes = 0
failures = 0
errors = 0
prompt_number = 1
for ws in nb.worksheets:
new_cells = []
for cell in ws.cells:
cell.prompt_number = prompt_number
if cell['cell_type'] != 'code':
new_cells.append(cell)
continue
if run_cells:
try:
outs = run_cell(kc, cell,
collect_outputs=modify_outputs,
timeout=timeout)
except Exception as e:
sys.stdout.write("failed to run cell:" + repr(e))
errors += 1
continue
sys.stdout.write('.')
if modify_outputs:
cell.outputs = outs
new_cell = cell_filter(cell)
if new_cell is not None:
new_cells.append(new_cell)
prompt_number += 1
sys.stdout.write('\n')
ws.cells = new_cells
km.shutdown_kernel()
del km
return nb
def __init__(self, notebook_dir, extra_args=None, profile=None,
timeout=90):
self.notebook_dir = os.path.abspath(notebook_dir)
self.profile = profile
self.timeout = timeout
km = KernelManager()
if extra_args is None:
extra_args = []
if profile is not None:
extra_args += ["--profile=%s" % profile]
km.start_kernel(stderr=open(os.devnull, 'w'),
extra_arguments=extra_args)
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError: # still on 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# make sure it's working
shell.execute("pass")
shell.get_msg()
# all of these should be run pylab inline
shell.execute("%pylab inline")
shell.get_msg()
self.kc = kc
self.km = km
self.iopub = iopub
def setup_client():
manager = KernelManager()
#manager.kernel_name = "spread_kernel"
manager.start_kernel()
#Setup the client
client = manager.client()
client.start_channels()
#Hijack the IOPub channel
new_io = MyIOpubChannel(client.context,client.session, client._make_url('iopub'))
client._iopub_channel = new_io
#Set up said channel
client.iopub_channel.start()
for method in client.iopub_channel.proxy_methods:
setattr(client, method, getattr(client.iopub_channel, method))
#Hijack the shell channel
#old_io_2 = client._shell_channel
new_shell = MyShellChannel(client.context,client.session, client._make_url('shell'))
client._shell_channel = new_shell
#Set up said channel
client.shell_channel.start()
for method in client.shell_channel.proxy_methods:
setattr(client, method, getattr(client.shell_channel, method))
return (manager,client)
def test_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'],
stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
failed = False
for out, ref in zip(outs, cell.outputs):
if not compare_outputs(out, ref):
failed = True
if failed:
failures += 1
else:
successes += 1
sys.stdout.write('.')
print
print "tested notebook %s" % nb.metadata.name
print " %3i cells successfully replicated" % successes
if failures:
print " %3i cells mismatched output" % failures
if errors:
print " %3i cells failed to complete" % errors
kc.stop_channels()
km.shutdown_kernel()
del km
def test_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'],
stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
failed = False
for out, ref in zip(outs, cell.outputs):
if not compare_outputs(out, ref):
failed = True
if failed:
failures += 1
else:
successes += 1
sys.stdout.write('.')
print
print "tested notebook %s" % nb.metadata.name
print " %3i cells successfully replicated" % successes
if failures:
print " %3i cells mismatched output" % failures
if errors:
print " %3i cells failed to complete" % errors
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb):
"""Run IPython Notebook.
Paramters:
----------
nb : IPython Notebook in JSON format.
Returns:
--------
ret : int
Return value; 0 in case of no failure, 1 otherwise
"""
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
reply = shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print "\nFAILURE:"
print cell.input
print '-----'
print "raised:"
print '\n'.join(reply['traceback'])
cells += 1
sys.stdout.write('.')
print
print "ran notebook %s" % nb.metadata.name
print " ran %3i cells" % cells
if failures:
print " %3i cells raised exceptions" % failures
kc.stop_channels()
km.shutdown_kernel()
del km
if failures:
return 1
return 0
def run_notebook(nb,
cell_filter=lambda cell: cell,
extra_arguments=['--pylab=inline', '--profile=stats'],
modify_outputs=True,
run_cells=True):
"""
Take a notebook and send all its cells to a kernel.
Takes an optional filter to modify the results of the
cell after being run and having its
output set by `run_cell` if modify_outputs is True.
"""
km = KernelManager()
km.start_kernel(extra_arguments=extra_arguments,
stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
successes = 0
failures = 0
errors = 0
prompt_number = 1
for ws in nb.worksheets:
new_cells = []
for cell in ws.cells:
cell.prompt_number = prompt_number
if cell['cell_type'] != 'code':
new_cells.append(cell)
continue
if run_cells:
try:
outs = run_cell(kc, cell, collect_outputs=modify_outputs)
except Exception as e:
sys.stdout.write("failed to run cell:" + repr(e))
errors += 1
continue
sys.stdout.write('.')
if modify_outputs:
cell.outputs = outs
new_cell = cell_filter(cell)
if new_cell is not None:
new_cells.append(new_cell)
prompt_number += 1
sys.stdout.write('\n')
ws.cells = new_cells
km.shutdown_kernel()
del km
return nb
def run_notebook(nb):
"""Run IPython Notebook.
Paramters:
----------
nb : IPython Notebook in JSON format.
Returns:
--------
ret : int
Return value; 0 in case of no failure, 1 otherwise
"""
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
reply = shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print "\nFAILURE:"
print cell.input
print '-----'
print "raised:"
print '\n'.join(reply['traceback'])
cells += 1
sys.stdout.write('.')
print
print "ran notebook %s" % nb.metadata.name
print " ran %3i cells" % cells
if failures:
print " %3i cells raised exceptions" % failures
kc.stop_channels()
km.shutdown_kernel()
del km
if failures:
return 1
return 0
def run_notebook(nb, save_output=False):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
if hasattr(km, 'client'):
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
else:
# IPython 0.13 compat
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
rendered_cells = list()
for cell in ws.cells:
rendered_cells.append(cell)
if cell.cell_type != 'code':
continue
outputs, failed, payload = run_cell(shell, iopub, cell)
cell.outputs = outputs
cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
# Very hacky code to execute the loaded Python code
if payload and payload[0]['source'] == 'set_next_input':
new_cell = cell.copy()
new_cell["input"] = payload[0]["text"]
outputs, failed, _ = run_cell(shell, iopub, new_cell)
new_cell.outputs = outputs
new_cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
rendered_cells.append(new_cell)
if save_output:
ws.cells = rendered_cells
print()
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready()
except AttributeError:
# IPython < 3
kc.kernel_info()
while True:
msg = kc.get_shell_msg(block=True, timeout=30)
if msg['msg_type'] == 'kernel_info_reply':
break
# Flush IOPub channel
while True:
try:
msg = kc.get_iopub_msg(block=True, timeout=0.2)
except Empty:
break
# simple ping:
kc.execute("pass")
kc.get_shell_msg()
cells = 0
failures = 0
if hasattr(nb, 'worksheets'):
# nobody uses more than 1 worksheet
ws = nb.worksheets[0]
else:
# no more worksheet level in new format
ws = nb
for cell in ws.cells:
if cell.cell_type != 'code':
continue
outputs, failed = run_cell(kc, cell)
cell.outputs = outputs
cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
sys.stdout.flush()
print()
print("ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready()
except AttributeError:
# IPython < 3
kc.kernel_info()
while True:
msg = kc.get_shell_msg(block=True, timeout=30)
if msg['msg_type'] == 'kernel_info_reply':
break
# Flush IOPub channel
while True:
try:
msg = kc.get_iopub_msg(block=True, timeout=0.2)
except Empty:
break
# simple ping:
kc.execute("pass")
kc.get_shell_msg()
cells = 0
failures = 0
if hasattr(nb, 'worksheets'):
# nobody uses more than 1 worksheet
ws = nb.worksheets[0]
else:
# no more worksheet level in new format
ws = nb
for cell in ws.cells:
if cell.cell_type != 'code':
continue
outputs, failed = run_cell(kc, cell)
cell.outputs = outputs
cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
sys.stdout.flush()
print()
print("ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb, output=False):
"""
"""
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
log.info('Run cell #%i' % cells)
cells += 1
outs = run_cell(shell, iopub, cell, output=output)
if outs:
for i in range(len(outs)):
if outs[i]['output_type'] == "pyerr":
log.error('Fail to execute cell #%i\n' % cells + '\n'.join(outs[i]['traceback']))
failures += 1
continue
log.info('Done')
log.info("%i cells runned with %i cells failed" % (cells, failures))
kc.stop_channels()
km.shutdown_kernel()
del km
def start_new_kernel():
"""start a new kernel, and return its Manager and Client"""
km = KernelManager()
km.start_kernel(stdout=PIPE, stderr=PIPE)
kc = km.client()
kc.start_channels()
msg_id = kc.kernel_info()
kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(kc)
return km, kc
def setup():
global KM, KC
KM = KernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KC = KM.client()
KC.start_channels()
# wait for kernel to be ready
KC.execute("pass")
KC.get_shell_msg(block=True, timeout=5)
flush_channels()
def setup():
global KM, KC
KM = KernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KC = KM.client()
KC.start_channels()
# wait for kernel to be ready
KC.execute("pass")
KC.get_shell_msg(block=True, timeout=5)
flush_channels()
def start_new_kernel():
"""start a new kernel, and return its Manager and Client"""
km = KernelManager()
km.start_kernel(stdout=PIPE, stderr=PIPE)
kc = km.client()
kc.start_channels()
msg_id = kc.kernel_info()
kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(kc)
return km, kc
def start_new_kernel(argv=None):
"""start a new kernel, and return its Manager and Client"""
km = KernelManager()
kwargs = dict(stdout=nose.iptest_stdstreams_fileno(), stderr=STDOUT)
if argv:
kwargs['extra_arguments'] = argv
km.start_kernel(**kwargs)
kc = km.client()
kc.start_channels()
msg_id = kc.kernel_info()
kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(kc)
return km, kc
def run_nb_offline(nb_path):
""" Read notebook from filepath and execute it; report errors in code cells.
"""
if not os.path.isfile(nb_path):
raise Exception('Invalid path: %s' % nb_path)
with open(nb_path) as f:
nb = reads(f.read(), 'json')
logging.info("Running notebook %s" % nb.metadata.name)
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code': #ONLY RUN CODE CELLS
continue
shell.execute(cell.input)
# wait for finish, maximum TIMEOUT
reply = shell.get_msg(timeout=MAX_TIMEOUT)['content']
if reply['status'] == 'error':
failures += 1
logging.info("\nNotebook FAILURE:")
logging.info(cell.input)
logging.info('-----')
logging.info('raised:')
logging.info('\n'.join(reply['traceback']))
cells += 1
# sys.stdout.write('.')
logging.info("Finished running notebook")
logging.info(" ran %3i cells" % cells)
if failures:
logging.warning(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def start_new_kernel(argv=None):
"""start a new kernel, and return its Manager and Client"""
km = KernelManager()
kwargs = dict(stdout=PIPE, stderr=PIPE)
if argv:
kwargs['extra_arguments'] = argv
km.start_kernel(**kwargs)
kc = km.client()
kc.start_channels()
msg_id = kc.kernel_info()
kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(kc)
return km, kc
def execute_kernel(nb, t, tshell):
"""
Load Kernel stuff
iopub may be necessary to run through the cell
"""
km = KernelManager()
km.start_kernel(extra_arguments=['--matplotlib=inline'],
stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
"""
This part needs revision
"""
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
"""
Try to print cell by cell
"""
# Only one worksheet in the current ipython nbs structure
for cell in nb.worksheets[0].cells:
# If the cell is code, move to next cell
if cell.cell_type != 'code':
continue
# Otherwise the cell is an output cell, run it!
try:
outs = run_cell(shell, iopub, cell, t, tshell)
print outs
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
def run_notebook(nb, pylab_inline=True, timeout=20):
"""
Run the notebook, populating the output cells with appropriate content.
Params
------
nb : the contents of a notebook as a string
pylab_inline : i.e. should the command be executed as if it was flagged with
--paylab=inline
timeout : the length of time in seconds to wait before the script is
considered timed out. I set this to a big value for some
data heavy scripts
"""
# Start the kernel.
km = KernelManager()
args = {}
if pylab_inline:
args['extra_arguments'] = ['--pylab=inline']
km.start_kernel(**args)
# Get our client.
try:
kc = km.client()
except AttributeError:
kc = km
kc.start_channels()
shell = kc.shell_channel
# Ping the kernel.
shell.execute('pass')
shell.get_msg()
# Run all the cells.
cells_executed, cells_failed = 0, 0
for ws in nb.worksheets:
for cell in ws.cells:
cell.prompt_number = cells_executed + 1
if cell.cell_type != 'code':
continue
cells_executed += 1
run_cell(kc, cell, timeout)
# Clean up resources. (Hopefully?)
kc.stop_channels()
km.shutdown_kernel()
del km
return cells_failed
def run_notebook(nb, pylab_inline=True, timeout=20):
"""
Run the notebook, populating the output cells with appropriate content.
Params
------
nb : the contents of a notebook as a string
pylab_inline : i.e. should the command be executed as if it was flagged with
--paylab=inline
timeout : the length of time in seconds to wait before the script is
considered timed out. I set this to a big value for some
data heavy scripts
"""
# Start the kernel.
km = KernelManager()
args = {}
if pylab_inline:
args['extra_arguments'] = ['--pylab=inline']
km.start_kernel(**args)
# Get our client.
try:
kc = km.client()
except AttributeError:
kc = km
kc.start_channels()
shell = kc.shell_channel
# Ping the kernel.
shell.execute('pass')
shell.get_msg()
# Run all the cells.
cells_executed, cells_failed = 0, 0
for ws in nb.worksheets:
for cell in ws.cells:
cell.prompt_number = cells_executed + 1
if cell.cell_type != 'code':
continue
cells_executed += 1
run_cell(kc, cell, timeout)
# Clean up resources. (Hopefully?)
kc.stop_channels()
km.shutdown_kernel()
del km
return cells_failed
def new_ipy(s=''):
"""Create a new IPython kernel (optionally with extra arguments)
XXX: Allow passing of profile information here
Examples
--------
new_ipy()
"""
from IPython.kernel import KernelManager
km = KernelManager()
km.start_kernel()
return km_from_string(km.connection_file)
def setup():
global KM, KC
KM = KernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KC = KM.client()
KC.start_channels()
# wait for kernel to be ready
try:
msg = KC.iopub_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT)
except Empty:
pass
msg_id = KC.kernel_info()
KC.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels()
def start_new_kernel(argv=None):
"""start a new kernel, and return its Manager and Client"""
km = KernelManager()
kwargs = dict(stdout=nose.ipy_stream_capturer.writefd, stderr=STDOUT)
if argv:
kwargs['extra_arguments'] = argv
km.start_kernel(**kwargs)
nose.ipy_stream_capturer.ensure_started()
kc = km.client()
kc.start_channels()
msg_id = kc.kernel_info()
kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(kc)
return km, kc
def setup():
global KM, KC
KM = KernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KC = KM.client()
KC.start_channels()
# wait for kernel to be ready
try:
msg = KC.iopub_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT)
except Empty:
pass
msg_id = KC.kernel_info()
KC.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels()
def run_nb_offline(nb_path):
""" Read notebook from filepath and execute it; report errors in code cells.
"""
if not os.path.isfile(nb_path):
raise Exception("Invalid path: %s" % nb_path)
with open(nb_path) as f:
nb = reads(f.read(), "json")
logging.info("Running notebook %s" % nb.metadata.name)
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, "w"))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != "code": # ONLY RUN CODE CELLS
continue
shell.execute(cell.input)
# wait for finish, maximum TIMEOUT
reply = shell.get_msg(timeout=MAX_TIMEOUT)["content"]
if reply["status"] == "error":
failures += 1
logging.info("\nNotebook FAILURE:")
logging.info(cell.input)
logging.info("-----")
logging.info("raised:")
logging.info("\n".join(reply["traceback"]))
cells += 1
# sys.stdout.write('.')
logging.info("Finished running notebook")
logging.info(" ran %3i cells" % cells)
if failures:
logging.warning(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def execute_kernel(nb, t, tshell):
"""
Load Kernel stuff
iopub may be necessary to run through the cell
"""
km = KernelManager()
km.start_kernel(extra_arguments=['--matplotlib=inline'],
stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
"""
This part needs revision
"""
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
"""
Try to print cell by cell
"""
# Only one worksheet in the current ipython nbs structure
for cell in nb.worksheets[0].cells:
# If the cell is code, move to next cell
if cell.cell_type != 'code':
continue
# Otherwise the cell is an output cell, run it!
try:
outs = run_cell(shell, iopub, cell, t, tshell)
print outs
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
class RunningKernel(object):
def __init__(self):
self.km = KernelManager()
self.km.start_kernel(stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.shell.execute("pass")
self.shell.get_msg()
def restart(self):
self.km.restart_kernel(now=True)
def stop(self):
self.kc.stop_channels()
self.km.shutdown_kernel()
del self.km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=['--profile', 'stats'])#, stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
shell.execute("datadir = '%s'" % os.path.abspath(os.path.join( \
os.path.abspath(os.path.dirname(__file__)), '..', 'data')))
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
msg = shell.get_msg(timeout=20)
reply = msg['content']
if reply['status'] == 'error':
failures += 1
print "\nFAILURE:"
print cell.input
print '-----'
print "raised:"
print '\n'.join(reply['traceback'])
cells += 1
sys.stdout.write('.')
print "ran notebook %s" % nb.metadata.name
print " ran %3i cells" % cells
if failures:
print " %3i cells raised exceptions" % failures
kc.stop_channels()
km.shutdown_kernel()
del km
class RunningKernel(object):
def __init__(self):
self.km = KernelManager()
self.km.start_kernel(stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.shell.execute("pass")
self.shell.get_msg()
def restart(self):
self.km.restart_kernel(now=True)
def stop(self):
self.kc.stop_channels()
self.km.shutdown_kernel()
del self.km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:")
print(cell.input)
print('-----')
print("raised:")
print('\n'.join(reply['traceback']))
print(cell)
cells += 1
sys.stdout.write('.')
print()
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:")
print(cell.input)
print('-----')
print("raised:")
print('\n'.join(reply['traceback']))
print(cell)
cells += 1
sys.stdout.write('.')
print()
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
if hasattr(km, 'client'):
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
else:
# IPython 0.13 compat
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
outputs, failed = run_cell(shell, iopub, cell)
cell.outputs = outputs
cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
sys.stdout.flush()
print()
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
if hasattr(km, 'client'):
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
else:
# IPython 0.13 compat
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
outputs, failed = run_cell(shell, iopub, cell)
cell.outputs = outputs
cell['prompt_number'] = cells
failures += failed
cells += 1
sys.stdout.write('.')
sys.stdout.flush()
print()
print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def new_kernel():
"""start a kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
KM = KernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KC = KM.client()
KC.start_channels()
# wait for kernel to be ready
KC.shell_channel.execute("import sys")
KC.shell_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(KC)
try:
yield KC
finally:
KC.stop_channels()
KM.shutdown_kernel()
def test_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=[], stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
shell = kc.shell_channel
shell.kernel_info()
while True:
try:
kc.iopub_channel.get_msg(timeout=1)
except Empty:
break
errors = 0
cells = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
cells += 1
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
print("failed to run cell:", repr(e))
print(cell.input)
errors += 1
continue
cell.outputs = outs
if errors:
print(" %3i cells failed to complete" % errors)
if cells:
print("%i code cells from notebook %s" % (cells, nb.metadata.name))
kc.stop_channels()
km.shutdown_kernel()
del km
def new_kernel():
"""start a kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
KM = KernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KC = KM.client()
KC.start_channels()
# wait for kernel to be ready
KC.shell_channel.execute("import sys")
KC.shell_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT)
flush_channels(KC)
try:
yield KC
finally:
KC.stop_channels()
KM.shutdown_kernel()
def test_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=[], stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
shell = kc.shell_channel
shell.kernel_info()
while True:
try:
kc.iopub_channel.get_msg(timeout=1)
except Empty:
break
errors = 0
cells = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
cells += 1
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
print("failed to run cell:", repr(e))
print(cell.input)
errors += 1
continue
cell.outputs = outs
if errors:
print(" %3i cells failed to complete" % errors)
if cells:
print("%i code cells from notebook %s" % (cells, nb.metadata.name))
kc.stop_channels()
km.shutdown_kernel()
del km
def execute_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=["--pylab=inline", "--profile=stats"], stderr=open(os.devnull, "w"))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
successes = 0
failures = 0
errors = 0
prompt_number = 1
for ws in nb.worksheets:
for cell in ws.cells:
cell.prompt_number = prompt_number
if cell.cell_type != "code":
continue
run_cell(kc, cell)
try:
outs = run_cell(kc, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
sys.stdout.write(".")
cell.outputs = outs
prompt_number += 1
km.shutdown_kernel()
del km
return nb
def test_noexceptions(nb_path):
"""Ensure that no cells raise an exception."""
with open(nb_path) as f:
nb = current.reads(f.read(), 'json')
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
# Simple ping
shell.execute("pass")
shell.get_msg()
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 2 minutes
reply = shell.get_msg(timeout=120)['content']
if reply['status'] == 'error':
err_msg = ("\nFAILURE:" + cell.input + "\n"
"-----\nraised:\n"
+ "\n".join(reply['traceback']))
kc.stop_channels()
km.shutdown_kernel()
del km
assert False, err_msg
kc.stop_channels()
km.shutdown_kernel() # noqa
del km # noqa
assert True
def run_notebook(nb):
"""
Run each code cell in a given notebook and update with the new output
"""
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'])
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
cell.outputs = run_cell(shell, iopub, cell)
except Exception as e:
return -1
kc.stop_channels()
km.shutdown_kernel()
del km
return nb
def run_notebook(nb):
"""
Run each code cell in a given notebook and update with the new output
"""
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'])
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
cell.outputs = run_cell(shell, iopub, cell)
except Exception as e:
return -1
kc.stop_channels()
km.shutdown_kernel()
del km
return nb
def execute_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline', '--profile=stats'], stderr=open(os.devnull, 'w'))
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
successes = 0
failures = 0
errors = 0
prompt_number = 1
for ws in nb.worksheets:
for cell in ws.cells:
cell.prompt_number = prompt_number
if cell.cell_type != 'code':
continue
run_cell(kc, cell)
try:
outs = run_cell(kc, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
sys.stdout.write('.')
cell.outputs = outs
prompt_number += 1
km.shutdown_kernel()
del km
return nb
def __init__(self, nb_in, pylab):
km = KernelManager()
if pylab:
km.start_kernel(extra_arguments=['--pylab=inline'])
else:
km.start_kernel()
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
sleep(1)
kc = km.client()
kc.start_channels()
self.shell = kc.shell_channel
self.iopub = kc.iopub_channel
logging.info('Reading notebook %s', nb_in)
self.nb = read(open(nb_in), 'json')
class IPyNbFile(pytest.File):
def collect(self):
with self.fspath.open() as f:
self.nb = reads(f.read(), 'json')
cell_num = 0
for ws in self.nb.worksheets:
for cell in ws.cells:
if cell.cell_type == "code":
yield IPyNbCell(self.name, self, cell_num, cell)
cell_num += 1
def setup(self):
self.km = KernelManager()
self.km.start_kernel(stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
def teardown(self):
self.km.shutdown_kernel()
del self.shell
del self.km
class IPyNbFile(pytest.File):
def collect(self):
with self.fspath.open() as f:
self.nb = reads(f.read(), 'json')
cell_num = 0
for ws in self.nb.worksheets:
for cell in ws.cells:
if cell.cell_type == "code":
yield IPyNbCell(self.name, self, cell_num, cell)
cell_num += 1
def setup(self):
self.km = KernelManager()
self.km.start_kernel(stderr=open(os.devnull, 'w'))
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
def teardown(self):
self.km.shutdown_kernel()
del self.shell
del self.km
def test_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
prompt_number = 1
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
cell.outputs = [e]
continue
failed = False
for out, ref in zip(outs, cell.outputs):
if not compare_outputs(out, ref):
failed = True
if failed:
failures += 1
else:
successes += 1
sys.stdout.write('.')
cell.outputs = outs
cell.prompt_number = prompt_number
if cell.outputs:
cell.outputs[0]['metadata'] = {}
prompt_number += 1
print
print "[{NAME}] {s} cells successfully replicated".format(NAME=NAME,
s=successes)
if failures:
print "[{NAME}] {f} cells mismatched output".format(NAME=NAME,
f=failures)
if errors:
print "[{NAME}] {e} cells failed to complete".format(NAME=NAME,
e=errors)
kc.stop_channels()
km.shutdown_kernel()
del km
def test_notebook(nb):
"""Main function to run tests at the level of one notebook."""
# Boot up the kernel, assume inline plotting
km = KernelManager()
km.start_kernel(
extra_arguments=["--matplotlib=inline", "--colors=NoColor"],
stderr=open(os.devnull, 'w'))
# Connect, allowing for older IPythons
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# Initialize the result tracking
successes = 0
failures = 0
errors = 0
fail_messages = []
err_messages = []
# Iterate the notebook, testing only code cells
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
# Try and get the prompt number for easier reference
try:
prompt_num = cell.prompt_number
except AttributeError:
prompt_num = None
# Try to execute the cell, catch errors from test execution
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
message = "Error while running cell:\n%s" % repr(e)
err_messages.append(message)
errors += 1
sys.stdout.write("E")
continue
errored = False
failed = False
for out, ref in zip(outs, cell.outputs):
# Now check for an error in the cell execution itself
bad_error = (out.output_type == "pyerr"
and not ref.output_type == "pyerr")
if bad_error:
message = "\nError in code cell"
if prompt_num is not None:
message = " %s (#%d)" % (message, prompt_num)
message = "%s:\n%s" % (message, "".join(out.traceback))
err_messages.append(message)
errored = True
# Otherwise check whether the stored and achived outputs match
else:
try:
match, message = compare_outputs(out, ref, prompt_num)
if not match:
failed = True
fail_messages.append(message)
except Exception as e:
message = "Error while comparing output:\n%s" % repr(e)
err_messages.append(message)
errors += 1
sys.stdout.write("E")
continue
if failed:
failures += 1
dot = "F"
elif errored:
errors += 1
dot = "E"
else:
successes += 1
dot = "."
print(dot, end="")
print()
print(" %3i cells successfully replicated" % successes)
if failures:
print(" %3i cells mismatched output" % failures)
if errors:
print(" %3i cells failed to complete" % errors)
if failures:
print("Failures:")
print("-" * 20)
print("\n" + "\n".join(fail_messages) + "\n")
if errors:
print("Errors:")
print("-" * 20)
print("\n" + "\n".join(err_messages) + "\n")
kc.stop_channels()
km.shutdown_kernel()
del km
return int(bool(failures + errors))
class IPyKernel(object):
"""
A simple wrapper class to run cells in an IPython Notebook.
Notes
-----
Use `with` construct to properly instantiate
"""
def __init__(self, console=None):
# default timeout time is 60 seconds
self.default_timeout = 60
self.extra_arguments = ['--pylab=inline']
def __enter__(self):
self.km = KernelManager()
self.km.start_kernel(extra_arguments=self.extra_arguments,
stderr=open(os.devnull, 'w'))
try:
self.kc = self.km.client()
self.kc.start_channels()
self.iopub = self.kc.iopub_channel
except AttributeError:
# IPython 0.13
self.kc = self.km
self.kc.start_channels()
self.iopub = self.kc.sub_channel
self.shell = self.kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
self.shell.execute("pass")
self.shell.get_msg()
while True:
try:
self.iopub.get_msg(timeout=1)
except Empty:
break
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.kc.stop_channels()
self.km.shutdown_kernel()
del self.km
def run(self, cell, timeout=None):
use_timeout = self.default_timeout
if timeout is not None:
use_timeout = timeout
self.shell.execute(cell.input)
self.shell.get_msg(timeout=use_timeout)
outs = []
while True:
try:
msg = self.iopub.get_msg(timeout=0.5)
except Empty:
break
msg_type = msg['msg_type']
if msg_type in ('status', 'pyin'):
continue
elif msg_type == 'clear_output':
outs = []
continue
content = msg['content']
out = NotebookNode(output_type=msg_type)
if msg_type == 'stream':
out.stream = content['name']
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
out['metadata'] = content['metadata']
for mime, data in content['data'].iteritems():
attr = mime.split('/')[-1].lower()
# this gets most right, but fix svg+html, plain
attr = attr.replace('+xml', '').replace('plain', 'text')
setattr(out, attr, data)
if msg_type == 'pyout':
out.prompt_number = content['execution_count']
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
else:
print "unhandled iopub msg:", msg_type
outs.append(out)
return outs
def sanitize(self, s):
"""sanitize a string for comparison.
fix universal newlines, strip trailing newlines, and normalize likely random values (memory addresses and UUIDs)
"""
if not isinstance(s, basestring):
return s
# normalize newline:
s = s.replace('\r\n', '\n')
# ignore trailing newlines (but not space)
s = s.rstrip('\n')
# normalize hex addresses:
s = re.sub(r'0x[a-f0-9]+', '0xFFFFFFFF', s)
# normalize UUIDs:
s = re.sub(r'[a-f0-9]{8}(\-[a-f0-9]{4}){3}\-[a-f0-9]{12}', 'U-U-I-D',
s)
return s
def compare_outputs(self,
test,
ref,
skip_compare=('png', 'traceback', 'latex',
'prompt_number', 'svg', 'html')):
for key in ref:
if key not in test:
# print "missing key: %s != %s" % (test.keys(), ref.keys())
return False
elif key not in skip_compare:
s1 = self.sanitize(test[key])
s2 = self.sanitize(ref[key])
if s1 != s2:
# print "mismatch %s:" % key
expected = s1.splitlines(1)
actual = s2.splitlines(1)
# diff=difflib.unified_diff(expected, actual)
# print ''.join(diff)
return False
return True
def get_commands(self, cell):
commands = {}
if hasattr(cell, 'input'):
lines = cell.input.splitlines()
if len(lines) > 0:
first_line = lines[0]
if first_line.startswith('#!'):
txt = first_line[2:].strip()
parts = txt.split(',')
for part in parts:
subparts = part.split(':')
if len(subparts) == 1:
commands[subparts[0].strip().lower()] = True
elif len(subparts) == 2:
commands[subparts[0].strip().lower()] = subparts[1]
return commands
class NotebookRunner(object):
# The kernel communicates with mime-types while the notebook
# uses short labels for different cell types. We'll use this to
# map from kernel types to notebook format types.
MIME_MAP = {
'image/jpeg': 'jpeg',
'image/png': 'png',
'text/plain': 'text',
'text/html': 'html',
'text/latex': 'latex',
'application/javascript': 'html',
}
def __init__(self, **kw):
"""Initializes the notebook runner.
Requires config rcloud_python_lib_path -- or raises exception"""
self.km = KernelManager()
self.km.kernel_cmd = make_ipkernel_cmd("""
import sys;
sys.path.extend("{RCPATH}".split(":"));
from rcloud_kernel import main;
main()""".format(RCPATH=kw["rcloud_python_lib_path"]), **kw)
del kw["rcloud_python_lib_path"]
self.km.client_factory = MyClient
self.km.start_kernel(**kw)
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
sleep(1)
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.iopub = self.kc.iopub_channel
def __del__(self):
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def shutdown(self):
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def run_cmd(self, cmd):
d = {}
class Cell(object):
def __init__(self):
self.input = cmd
def __setitem__(self, k, v):
d[k] = v
self.run_cell(Cell())
return d['outputs']
def run_magic(self, magic_line):
self.shell.magic(magic_line)
def wait_for_msg(self):
outs = list()
reply = self.shell.get_msg()
status = reply['content']['status']
if status == 'error':
logging.info('Cell raised uncaught exception: \n%s', '\n'.join(reply['content']['traceback']))
else:
if _debugging: logging.debug('Cell returned')
while True:
try:
msg = self.iopub.get_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
raise
outs.append(msg)
return outs
def run_cell(self, cell):
'''
Run a notebook cell and update the output of that cell in-place.
'''
if _debugging: logging.debug('Running cell:\n%s\n', cell.input)
self.shell.execute(cell.input)
reply = self.shell.get_msg()
status = reply['content']['status']
if status == 'error':
logging.info('Cell raised uncaught exception: \n%s', '\n'.join(reply['content']['traceback']))
else:
if _debugging: logging.debug('Cell returned')
outs = list()
while True:
try:
msg = self.iopub.get_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
logging.warn('Execution state did not return to idle')
raise
content = msg['content']
msg_type = msg['msg_type']
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
cell['prompt_number'] = content['execution_count']
out.prompt_number = content['execution_count']
if msg_type in ['status', 'pyin']:
continue
elif msg_type == 'stream':
out.stream = content['name']
out.html = RClansiconv(content['data'])
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
logging.warning('unhandled mime type: %s' % mime)
raise NotImplementedError('unhandled mime type: %s' % mime)
if attr == 'text':
if _debugging: logging.info(data)
setattr(out, attr, RClansiconv(data))
else:
setattr(out, attr, data)
elif msg_type == 'pyerr':
logging.info('Received an exception: ' + content['ename'])
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
out.html = RClansiconv('\n'.join(out.traceback))
else:
raise NotImplementedError('unhandled iopub message: %s' % msg_type)
outs.append(out)
cell['outputs'] = outs
def iter_code_cells(self, nb):
'''
Iterate over the notebook cells containing code.
'''
nb = read(open(nb), 'json')
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
yield cell
def run_notebook(self, nb, skip_exceptions=False):
'''
Run all the cells of a notebook in order and update
the outputs in-place.
If ``skip_exceptions`` is set, then if exceptions occur in a cell, the
subsequent cells are run (by default, the notebook execution stops).
'''
for cell in self.iter_code_cells(nb):
try:
self.run_cell(cell)
print cell.outputs
except NotebookError:
if not skip_exceptions:
raise
def save_notebook(self, nb_out):
if _debugging: logging.info('Saving to %s', nb_out)
write(self.nb, open(nb_out, 'w'), 'json')
class NotebookRunner(object):
# The kernel communicates with mime-types while the notebook
# uses short labels for different cell types. We'll use this to
# map from kernel types to notebook format types.
MIME_MAP = {
'image/jpeg': 'jpeg',
'image/png': 'png',
'text/plain': 'text',
'text/html': 'html',
'text/latex': 'latex',
'application/javascript': 'html',
'application/json': 'json',
'image/svg+xml': 'svg',
}
def __init__(
self,
nb,
pylab=False,
mpl_inline=False,
profile_dir=None,
working_dir=None):
self.km = KernelManager()
args = []
if pylab:
args.append('--pylab=inline')
logging.warn(
'--pylab is deprecated and will be removed in a future version'
)
elif mpl_inline:
args.append('--matplotlib=inline')
logging.warn(
'--matplotlib is deprecated and' +
' will be removed in a future version'
)
if profile_dir:
args.append('--profile-dir=%s' % os.path.abspath(profile_dir))
cwd = os.getcwd()
if working_dir:
os.chdir(working_dir)
self.km.start_kernel(extra_arguments=args)
os.chdir(cwd)
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
sleep(1)
self.kc = self.km.client()
self.kc.start_channels()
try:
self.kc.wait_for_ready()
except AttributeError:
# IPython < 3
self._wait_for_ready_backport()
self.nb = nb
def shutdown_kernel(self):
logging.info('Shutdown kernel')
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def _wait_for_ready_backport(self):
# Backport BlockingKernelClient.wait_for_ready from IPython 3.
# Wait for kernel info reply on shell channel.
self.kc.kernel_info()
while True:
msg = self.kc.get_shell_msg(block=True, timeout=30)
if msg['msg_type'] == 'kernel_info_reply':
break
# Flush IOPub channel
while True:
try:
msg = self.kc.get_iopub_msg(block=True, timeout=0.2)
except Empty:
break
def run_cell(self, cell):
"""Run a notebook cell and update the output of that cell in-place."""
logging.info('Running cell:\n%s\n', cell.input)
self.kc.execute(cell.input)
reply = self.kc.get_shell_msg()
status = reply['content']['status']
traceback_text = ''
if status == 'error':
traceback_text = 'Cell raised uncaught exception: \n' + \
'\n'.join(reply['content']['traceback'])
logging.info(traceback_text)
else:
logging.info('Cell returned')
outs = list()
while True:
try:
msg = self.kc.get_iopub_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle
# before the queue becomes empty,
# if it doesn't, something bad has happened
raise
content = msg['content']
msg_type = msg['msg_type']
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format
# but uses error/execute_result in the message spec. This does the
# translation needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
cell['prompt_number'] = content['execution_count']
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
# in msgspec 5, this is name, text
# in msgspec 4, this is name, data
if 'text' in content:
out.text = content['text']
else:
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
raise NotImplementedError(
'unhandled mime type: %s' % mime
)
# In notebook version <= 3 JSON data is stored as a string
# Evaluation of IPython2's JSON gives strings directly
# Therefore do not encode for IPython versions prior to 3
json_encode = (
IPython.version_info[0] >= 3 and
mime == "application/json")
data_out = data if not json_encode else json.dumps(data)
setattr(out, attr, data_out)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = list()
continue
else:
raise NotImplementedError(
'unhandled iopub message: %s' % msg_type
)
outs.append(out)
cell['outputs'] = outs
logging.info('Cell output:\n%s\n', outs)
if status == 'error':
raise NotebookError(traceback_text)
def iter_code_cells(self):
"""Iterate over the notebook cells containing code."""
for ws in self.nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
yield cell
def run_notebook(self, skip_exceptions=False, progress_callback=None, cells=None):
"""
Run all the notebook cells in order and update the outputs in-place.
If ``skip_exceptions`` is set, then if exceptions occur in a cell, the
subsequent cells are run (by default, the notebook execution stops).
"""
for i, cell in enumerate(self.iter_code_cells()):
if cells is not None:
if i not in cells:
continue
try:
self.run_cell(cell)
except NotebookError:
if not skip_exceptions:
raise
if progress_callback:
progress_callback(i)
def count_code_cells(self):
"""Return the number of code cells in the notebook."""
return sum(1 for _ in self.iter_code_cells())
class NotebookRunner(object):
# The kernel communicates with mime-types while the notebook
# uses short labels for different cell types. We'll use this to
# map from kernel types to notebook format types.
MIME_MAP = {
'image/jpeg': 'jpeg',
'image/png': 'png',
'text/plain': 'text',
'text/html': 'html',
'text/latex': 'latex',
'application/javascript': 'html',
}
def __init__(self, **kw):
"""Initializes the notebook runner.
Requires config rcloud_python_lib_path -- or raises exception"""
self.km = KernelManager()
self.km.kernel_cmd = make_ipkernel_cmd(
"""
import sys;
import time; # to test for slow staring kernels, add a delay here
sys.path.extend("{RCPATH}".split(":"));
from rcloud_kernel import main;
main()""".format(RCPATH=kw["rcloud_python_lib_path"]), **kw)
del kw["rcloud_python_lib_path"]
self.km.client_factory = MyClient
self.km.start_kernel(
**kw) # This is a non-blocking call to launch the process
self.completer = completer.Completer()
self.completer.limit_to__all__ = True
# There is a possible race condition if the system is slow to load
# the modules for the kernel and we issue commands to the kernel rightaway.
# We saw these on CentOS and OSX. So, issue an empty command to the kernel and wait for return
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.shell.execute("")
_ = self.shell.get_msgs()
self.iopub = self.kc.iopub_channel
def __del__(self):
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def shutdown(self):
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def complete(self, text, pos_fromR):
"""Completions for text at pos"""
txtArr = text[:int(pos_fromR)].split('\n')
# We are ignoring surrounding lines and full context; a poor-man's tab-complete for now
_ = allres = self.shell.get_msgs() # flush before execution
self.shell.complete(txtArr[-1],
line="",
cursor_pos=len(txtArr[-1]) - 1)
while True:
resp = self.shell.get_msg()
if _debugging: logging.debug(resp)
if resp['msg_type'] == 'complete_reply':
_ = self.shell.get_msgs() # flush
return resp['content']['matches']
def run_cmd(self, cmd):
self.submit_cell(cmd)
def run_magic(self, magic_line):
self.shell.magic(magic_line)
def submit_cell(self, cmd):
"""Submits the cell code to the kernel.
Now that we have a continuous IO on rcloud, and the clients can send oob messages to
RCloud [but only through R], we separate the pieces to submit and have a function
for looping to get messages"""
if _debugging: logging.debug('Running cell:\n%s\n', cmd)
_ = self.iopub.flush()
self.shell.execute(cmd)
self._previous_status = 'START'
def poll_for_msgs(self):
"""Polls for messages from the kernel.
Used after submitting code for execution"""
try:
msg = self.iopub.get_msg(timeout=1)
if msg['msg_type'] == 'status' and msg['content'][
'execution_state'] == 'idle':
if _debugging:
logging.info('Message -- {}:{}'.format(
msg['msg_type'], msg['content']))
self._previous_status = 'IDLE'
return NotebookNode(output_type='IDLE')
except Empty: # state should return to idle before queue becomes empty, but we ignore it now
prevstat, self._previous_status = self._previous_status, 'EMPTY'
retstat = 'END_CELL' if prevstat == 'IDLE' else 'EMPTY'
# Assuming IDLE followed by EMPTY is the end-of-cell
return NotebookNode(output_type=retstat)
self._previous_status = '' # Not idle, that's all we are concerned about for now
content, msg_type = msg['content'], msg['msg_type']
if msg_type in ['status', 'pyin']:
return NotebookNode(output_type='NoOp')
out = NotebookNode(output_type=msg_type)
if msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
tmpval = RClansiconv(data) if attr == 'text' else data
setattr(out, attr, tmpval)
except KeyError:
raise NotImplementedError('unhandled mime type: %s' % mime)
elif msg_type == 'stream':
setattr(out, 'text', RClansiconv(content['data']))
elif msg_type == 'pyerr':
setattr(out, 'html',
RClansiconv('\n'.join(content['traceback']) + '\n'))
else:
if _debugging: logging.info('Unsupported: ' + msg_type)
raise NotImplementedError('unhandled iopub message: %s' % msg_type)
if _debugging:
logging.info(
'Sending: msg_type: [{}]; HTML: [{}]; TEXT: [{}]'.format(
msg_type, out.get('html', ''), out.get('text', '')))
return out # upstream process will handle it [e.g. send as an oob message]
class NotebookRunner(object):
# The kernel communicates with mime-types while the notebook
# uses short labels for different cell types. We'll use this to
# map from kernel types to notebook format types.
MIME_MAP = {
'image/jpeg': 'jpeg',
'image/png': 'png',
'text/plain': 'text',
'text/html': 'html',
'text/latex': 'latex',
'application/javascript': 'html',
'image/svg+xml': 'svg',
}
def __init__(self, nb, working_dir=None):
from IPython.kernel import KernelManager
self.km = KernelManager()
cwd = os.getcwd()
if working_dir is not None:
os.chdir(working_dir)
self.km.start_kernel()
os.chdir(cwd)
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
time.sleep(1)
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.iopub = self.kc.iopub_channel
self.nb = nb
def __del__(self):
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def run_cell(self, cell): # noqa: C901
"""Run a notebook cell and update the output of that cell in-place."""
self.shell.execute(cell.input)
reply = self.shell.get_msg()
status = reply['content']['status']
if status == 'error':
traceback_text = ("Cell raised uncaught exception: \n"
"\n".join(reply['content']['traceback']))
outs = []
while True:
msg = self.iopub.get_msg(timeout=1)
msg_type = msg['msg_type']
content = msg['content']
if msg_type == 'status' and content['execution_state'] == 'idle':
break
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format
# but uses error/execute_result in the message spec. This does
# the translation needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout',
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
cell['prompt_number'] = content['execution_count']
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
out.text = content['data']
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
raise NotImplementedError(
'unhandled mime type: %s' % mime)
setattr(out, attr, data)
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
elif msg_type == 'clear_output':
outs = []
continue
else:
raise NotImplementedError(
'unhandled iopub message: %s' % msg_type)
outs.append(out)
cell['outputs'] = outs
if status == 'error':
raise Exception(traceback_text)
def iter_code_cells(self):
"""Iterate over the notebook cells containing code."""
for ws in self.nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
yield cell
def run_notebook(self, skip_exceptions=False, progress_callback=None):
"""Runs all notebook cells in order and updates outputs in-place.
If ``skip_exceptions`` is True, then if exceptions occur in a cell, the
subsequent cells are run (by default, the notebook execution stops).
"""
for i, cell in enumerate(self.iter_code_cells()):
try:
self.run_cell(cell)
except:
if not skip_exceptions:
raise
if progress_callback is not None:
progress_callback(i)
def count_code_cells(self):
"""Return the number of code cells in the notebook."""
return sum(1 for _ in self.iter_code_cells())
class NotebookRunner(object):
# The kernel communicates with mime-types while the notebook
# uses short labels for different cell types. We'll use this to
# map from kernel types to notebook format types.
MIME_MAP = {
'image/jpeg': 'jpeg',
'image/png': 'png',
'text/plain': 'text',
'text/html': 'html',
'text/latex': 'latex',
'application/javascript': 'html',
'image/svg+xml': 'svg',
}
def __init__(self, nb, pylab=False, mpl_inline=False, working_dir = None):
self.km = KernelManager()
args = []
if pylab:
args.append('--pylab=inline')
logging.warn('--pylab is deprecated and will be removed in a future version')
elif mpl_inline:
args.append('--matplotlib=inline')
logging.warn('--matplotlib is deprecated and will be removed in a future version')
cwd = os.getcwd()
if working_dir:
os.chdir(working_dir)
self.km.start_kernel(extra_arguments = args)
os.chdir(cwd)
if platform.system() == 'Darwin':
# There is sometimes a race condition where the first
# execute command hits the kernel before it's ready.
# It appears to happen only on Darwin (Mac OS) and an
# easy (but clumsy) way to mitigate it is to sleep
# for a second.
sleep(1)
self.kc = self.km.client()
self.kc.start_channels()
self.shell = self.kc.shell_channel
self.iopub = self.kc.iopub_channel
self.nb = nb
def __del__(self):
self.kc.stop_channels()
self.km.shutdown_kernel(now=True)
def run_cell(self, cell):
'''
Run a notebook cell and update the output of that cell in-place.
'''
logging.info('Running cell:\n%s\n', cell.input)
self.shell.execute(cell.input)
reply = self.shell.get_msg()
status = reply['content']['status']
if status == 'error':
traceback_text = 'Cell raised uncaught exception: \n' + \
'\n'.join(reply['content']['traceback'])
logging.info(traceback_text)
else:
logging.info('Cell returned')
outs = list()
while True:
try:
msg = self.iopub.get_msg(timeout=1)
if msg['msg_type'] == 'status':
if msg['content']['execution_state'] == 'idle':
break
except Empty:
# execution state should return to idle before the queue becomes empty,
# if it doesn't, something bad has happened
raise
content = msg['content']
msg_type = msg['msg_type']
# IPython 3.0.0-dev writes pyerr/pyout in the notebook format but uses
# error/execute_result in the message spec. This does the translation
# needed for tests to pass with IPython 3.0.0-dev
notebook3_format_conversions = {
'error': 'pyerr',
'execute_result': 'pyout'
}
msg_type = notebook3_format_conversions.get(msg_type, msg_type)
out = NotebookNode(output_type=msg_type)
if 'execution_count' in content:
cell['prompt_number'] = content['execution_count']
out.prompt_number = content['execution_count']
if msg_type in ('status', 'pyin', 'execute_input'):
continue
elif msg_type == 'stream':
out.stream = content['name']
out.text = content['data']
#print(out.text, end='')
elif msg_type in ('display_data', 'pyout'):
for mime, data in content['data'].items():
try:
attr = self.MIME_MAP[mime]
except KeyError:
raise NotImplementedError('unhandled mime type: %s' % mime)
setattr(out, attr, data)
#print(data, end='')
elif msg_type == 'pyerr':
out.ename = content['ename']
out.evalue = content['evalue']
out.traceback = content['traceback']
#logging.error('\n'.join(content['traceback']))
elif msg_type == 'clear_output':
outs = list()
continue
else:
raise NotImplementedError('unhandled iopub message: %s' % msg_type)
outs.append(out)
cell['outputs'] = outs
if status == 'error':
raise NotebookError(traceback_text)
def iter_code_cells(self):
'''
Iterate over the notebook cells containing code.
'''
for ws in self.nb.worksheets:
for cell in ws.cells:
if cell.cell_type == 'code':
yield cell
def run_notebook(self, skip_exceptions=False, progress_callback=None):
'''
Run all the cells of a notebook in order and update
the outputs in-place.
If ``skip_exceptions`` is set, then if exceptions occur in a cell, the
subsequent cells are run (by default, the notebook execution stops).
'''
for i, cell in enumerate(self.iter_code_cells()):
try:
self.run_cell(cell)
except NotebookError:
if not skip_exceptions:
raise
if progress_callback:
progress_callback(i)
def count_code_cells(self):
'''
Return the number of code cells in the notebook
'''
return sum(1 for _ in self.iter_code_cells())
def test_notebook(nb):
km = KernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w'))
try:
kc = km.client()
kc.start_channels()
iopub = kc.iopub_channel
except AttributeError:
# IPython 0.13
kc = km
kc.start_channels()
iopub = kc.sub_channel
shell = kc.shell_channel
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
shell.execute("pass")
shell.get_msg()
while True:
try:
iopub.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
"""
IPython notebooks currently only support a single worksheet
Future versions may use many, so we leave the outer loop
A worksheet is simply a group of cells
e.g. could use for cell in nb.worksheets[0].cells:
"""
for ws in nb.worksheets:
for cell in ws.cells:
# If the cell is code, move to next cell
if cell.cell_type != 'code':
continue
# Otherwise the cell is an output cell, run it!
try:
outs = run_cell(shell, iopub, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
failed = False
for out, ref in zip(outs, cell.outputs):
# print '\n This is the output: \n'
# print out
# print '\n'
# print 'This is the reference:\n'
# print ref
# print '\n'
if not compare_outputs(out, ref):
failed = True
if failed:
failures += 1
else:
successes += 1
sys.stdout.write('.')
print
print "tested notebook %s" % nb.metadata.name
print bcolors.OKGREEN + " %3i cells successfully replicated" % + successes + bcolors.ENDC
if failures:
print bcolors.FAIL + " %3i cells mismatched output" % failures + bcolors.ENDC
if errors:
print bcolors.WARNING + " %3i cells failed to complete" % errors + bcolors.ENDC
kc.stop_channels()
km.shutdown_kernel()
del km
