def get_kernel(self):
security_dir = expanduser('~/.ipython/profile_default/security')
files = path(security_dir).files()
files.sort(key=lambda f: f.mtime,
reverse=True) # find last opened ipython session
def check_port(ip, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((ip, int(port)))
s.shutdown(2)
return True
except:
print 'failed to connect to %s:%s' % (ip, port)
return False
for file in files:
cfg = json.loads(open(file).read())
ip, port = cfg['ip'], cfg['shell_port']
if not check_port(ip, port): continue
kernel = BlockingKernelManager(**cfg)
kernel.start_channels()
kernel.shell_channel.session.key = kernel.key
kernel.hb_channel.unpause()
return kernel
print "Did not find a running IPython console."
print "Disabling IPython support..."
self.disabled = True
#raise Exception('Did not find a running IPython console.')
return None
def _launch_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel, kernel_manager: Popen instance and connected KernelManager
"""
kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE, env=env)
connection_file = os.path.join(IPYTHONDIR,
'profile_default',
'security',
'kernel-%i.json' % kernel.pid
)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + 5:
time.sleep(0.1)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise IOError("Connection file %r never arrived" % connection_file)
if kernel.poll() is not None:
raise IOError("Kernel failed to start")
km = BlockingKernelManager(connection_file=connection_file)
km.load_connection_file()
km.start_channels()
return kernel, km
def execute_notebook(nb):
km = BlockingKernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w'))
km.start_channels()
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
km.shell_channel.execute("pass")
km.shell_channel.get_msg()
while True:
try:
km.sub_channel.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
prompt_number = 1
for ws in nb.worksheets:
for cell in ws.cells:
cell.prompt_number = prompt_number
if cell.cell_type != 'code':
continue
try:
outs = run_cell(km, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
cell.outputs = outs
prompt_number += 1
km.shutdown_kernel()
del km
def run_notebook(nb):
km = BlockingKernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
km.start_channels()
shell = km.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print "\nFAILURE:"
print cell.input
print '-----'
print "raised:"
print '\n'.join(reply['traceback'])
cells += 1
sys.stdout.write('.')
print
print "ran notebook %s" % nb.metadata.name
print " ran %3i cells" % cells
if failures:
print " %3i cells raised exceptions" % failures
km.shutdown_kernel()
del km
def get_kernel(self):
security_dir = expanduser('~/.ipython/profile_default/security')
files = path(security_dir).files()
files.sort(key=lambda f: f.mtime, reverse=True) # find last opened ipython session
def check_port(ip, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect( (ip, int(port)) )
s.shutdown(2)
return True
except:
print 'failed to connect to %s:%s' % (ip, port)
return False
for file in files:
cfg = json.loads(open(file).read())
ip, port = cfg['ip'], cfg['shell_port']
if not check_port(ip, port): continue
kernel = BlockingKernelManager(**cfg)
kernel.start_channels()
kernel.shell_channel.session.key = kernel.key
kernel.hb_channel.unpause()
return kernel
print "Did not find a running IPython console."
print "Disabling IPython support..."
self.disabled = True
#raise Exception('Did not find a running IPython console.')
return None
def run_notebook(nb):
km = BlockingKernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
km.start_channels()
shell = km.shell_channel
# simple ping:
shell.execute("pass")
shell.get_msg()
cells = 0
failures = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
shell.execute(cell.input)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print "\nFAILURE:"
print cell.input
print '-----'
print "raised:"
print '\n'.join(reply['traceback'])
cells += 1
sys.stdout.write('.')
print
print "ran notebook %s" % nb.metadata.name
print " ran %3i cells" % cells
if failures:
print " %3i cells raised exceptions" % failures
km.shutdown_kernel()
del km
def __init__(self, *args, **kwargs):
connection_file = find_connection_file(kwargs.pop("connection_file"))
km = BlockingKernelManager(connection_file=connection_file)
km.load_connection_file()
heartbeat = True
km.start_channels(hb=heartbeat)
atexit.register(km.cleanup_connection_file)
super(IPythonConsoleShell, self).__init__(kernel_manager=km)
self.km = km
def __init__(self, *args, **kwargs):
connection_file = find_connection_file(kwargs.pop("connection_file"))
km = BlockingKernelManager(connection_file=connection_file)
km.load_connection_file()
heartbeat = True
km.start_channels(hb=heartbeat)
atexit.register(km.cleanup_connection_file)
super(IPythonConsoleShell, self).__init__(kernel_manager = km)
self.km = km
def km_from_string(s = PMX_IPYTHON_CONNECTION):
"""create kernel manager from IPKernelApp string
such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
or just 'kernel-12345.json' for IPython 0.12
"""
from os.path import join as pjoin
from IPython.zmq.blockingkernelmanager import BlockingKernelManager
from IPython.config.loader import KeyValueConfigLoader
from IPython.zmq.kernelapp import kernel_aliases
s = s.replace('--existing', '')
if 'connection_file' in BlockingKernelManager.class_trait_names():
from IPython.lib.kernel import find_connection_file
# 0.12 uses files instead of a collection of ports
# include default IPython search path
# filefind also allows for absolute paths, in which case the search
# is ignored
try:
# XXX: the following approach will be brittle, depending on what
# connection strings will end up looking like in the future, and
# whether or not they are allowed to have spaces. I'll have to sync
# up with the IPython team to address these issues -pi
if '--profile' in s:
k,p = s.split('--profile')
k = k.lstrip().rstrip() # kernel part of the string
p = p.lstrip().rstrip() # profile part of the string
fullpath = find_connection_file(k,p)
else:
fullpath = find_connection_file(s.lstrip().rstrip())
except IOError as e:
print(":IPython " + s + " failed", "Info")
print("^-- failed '" + s + "' not found", "Error")
return
km = BlockingKernelManager(connection_file = fullpath)
km.load_connection_file()
else:
if s == '':
print(":IPython 0.11 requires the full connection string")
return
loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
cfg = loader.load_config()['KernelApp']
try:
km = BlockingKernelManager(
shell_address=(ip, cfg['shell_port']),
sub_address=(ip, cfg['iopub_port']),
stdin_address=(ip, cfg['stdin_port']),
hb_address=(ip, cfg['hb_port']))
except KeyError as e:
print(":IPython " +s + " failed", "Info")
print("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
return
km.start_channels()
return km
def test_notebook(nb):
km = BlockingKernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w'))
km.start_channels()
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
km.shell_channel.execute("pass")
km.shell_channel.get_msg()
while True:
try:
km.sub_channel.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
outs = run_cell(km, cell)
except Exception as e:
print "failed to run cell:", repr(e)
print cell.input
errors += 1
continue
failed = False
for out, ref in zip(outs, cell.outputs):
if not compare_outputs(out, ref):
failed = True
if failed:
failures += 1
else:
successes += 1
sys.stdout.write('.')
print
print "tested notebook %s" % nb.metadata.name
print " %3i cells successfully replicated" % successes
if failures:
print " %3i cells mismatched output" % failures
if errors:
print " %3i cells failed to complete" % errors
km.shutdown_kernel()
del km
class IPythonConnection(object):
""" connects to a running IPython Kernel
and can execute code there
example usage:
1. start the IPython Kernel by typing::
ipython kernel
[IPKernelApp] To connect another client to this kernel, use:
[IPKernelApp] --existing kernel-4933.json
or even more useful to see the output::
ipython qtconsol
[IPKernelApp] To connect another client to this kernel, use:
[IPKernelApp] --existing kernel-4933.json
in a shell which will start the IPython Kernel and gives you
information on how to connect to it, you will need that immedeately.
2. create a IPythonConnection with the value from before::
ipc = IPythonConnection(4933)
3. now you can execute code like::
ipc.run_cell('print("Hello World")')
"""
def __init__(self,connection):
self.cf = find_connection_file(connection)
self.km = BlockingKernelManager(connection_file=self.cf)
self.km.load_connection_file()
self.km.start_channels()
def run_cell(self,code):
#self.shell = self.km.shell_channel
self.shell.execute(code)
def new_kernel():
"""start a kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
KM = BlockingKernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KM.start_channels()
# wait for kernel to be ready
KM.shell_channel.execute("import sys")
KM.shell_channel.get_msg(block=True, timeout=5)
flush_channels(KM)
try:
yield KM
finally:
KM.stop_channels()
KM.shutdown_kernel()
def new_kernel():
"""start a kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
KM = BlockingKernelManager()
KM.start_kernel(stdout=PIPE, stderr=PIPE)
KM.start_channels()
# wait for kernel to be ready
KM.shell_channel.execute("import sys")
KM.shell_channel.get_msg(block=True, timeout=5)
flush_channels(KM)
try:
yield KM
finally:
KM.stop_channels()
KM.shutdown_kernel()
class IPyNbFile(pytest.File):
def collect(self):
with self.fspath.open() as f:
self.nb = reads(f.read(), 'json')
cell_num = 0
for ws in self.nb.worksheets:
for cell in ws.cells:
if cell.cell_type == "code":
yield IPyNbCell(self.name, self, cell_num, cell)
cell_num += 1
def setup(self):
self.km = BlockingKernelManager()
self.km.start_kernel(stderr=open(os.devnull, 'w'))
self.km.start_channels()
self.shell = self.km.shell_channel
def teardown(self):
self.km.shutdown_kernel()
del self.shell
del self.km
def setup_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE, env=env)
connection_file = os.path.join(IPYTHONDIR,
'profile_default',
'security',
'kernel-%i.json' % kernel.pid
)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + 10:
time.sleep(0.1)
if kernel.poll() is not None:
o,e = kernel.communicate()
e = py3compat.cast_unicode(e)
raise IOError("Kernel failed to start:\n%s" % e)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise IOError("Connection file %r never arrived" % connection_file)
km = BlockingKernelManager(connection_file=connection_file)
km.load_connection_file()
km.start_channels()
try:
yield km
finally:
km.stop_channels()
kernel.terminate()
if s == '':
print ":IPython 0.11 requires the full connection string"
return
loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
cfg = loader.load_config()['KernelApp']
try:
km = BlockingKernelManager(
shell_address=(ip, cfg['shell_port']),
sub_address=(ip, cfg['iopub_port']),
stdin_address=(ip, cfg['stdin_port']),
hb_address=(ip, cfg['hb_port']))
except KeyError,e:
print ":IPython " +s + " failed", "Info"
print "^-- failed --"+e.message.replace('_port','')+" not specified", "Error"
return
km.start_channels()
return km
kernelManager = km_from_string()
def get_child_msg(msg_id):
# XXX: message handling should be split into its own process in the future
while True:
# get_msg will raise with Empty exception if no messages arrive in 1 second
m = kernelManager.shell_channel.get_msg(timeout = 1)
if m['parent_header']['msg_id'] == msg_id:
break
else:
#got a message, but not the one we were looking for
print 'skipping a message on shell_channel','WarningMsg'
return m
def test_notebook(nb, halt_on_error=True, km=None):
our_kernel = km is None
if our_kernel:
km = BlockingKernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w'))
km.start_channels()
print km.connection_file
# run %pylab inline, because some notebooks assume this
# even though they shouldn't
km.shell_channel.execute("pass")
km.shell_channel.get_msg()
km.restart_kernel()
km.shell_channel.get_msg()
while True:
try:
km.sub_channel.get_msg(timeout=1)
except Empty:
break
successes = 0
failures = 0
errors = 0
halted = False
for ws in nb.worksheets:
for cell in ws.cells:
if cell.cell_type != 'code':
continue
try:
outs, exec_count = run_cell(km, cell)
except Exception as e:
log.critical( "failed to run cell:"+ repr(e))
log.critical("tested notebook %s" % nb.metadata.name)
print cell.input
errors += 1
continue
c = '.'
for out in outs:
if out['output_type'] == 'pyerr':
c = 'E'
if log.getEffectiveLevel() < logging.FATAL:
sys.stderr.write(c)
failed = False
#for out, ref in zip(outs, cell.outputs):
# if not compare_outputs(out, ref):
# failed = True
if failed:
failures += 1
else:
successes += 1
cell.outputs = outs
cell.prompt_number = exec_count
if c == 'E' and halt_on_error:
log.info("halting on error")
halted = True
# XXX: this next part is specific to nbexplode
save_intermediate(km, ws, cell, nb.metadata.name)
break;
if log.getEffectiveLevel() <= logging.WARNING:
sys.stderr.write('\n')
log.info("tested notebook %s" % nb.metadata.name)
log.info(" %3i cells successfully replicated" % successes)
if failures:
log.info(" %3i cells mismatched output" % failures)
if errors:
log.info(" %3i cells failed to complete" % errors)
if our_kernel:
km.shutdown_kernel()
del km
return nb, halted
help="Try to run all cells (by default will halt on first error)")
parser.add_argument('-A', '--allnotebooks', '--all-notebooks',
default=False, action='store_true',
help="Try to run all notebook (by default will halt on first error)")
args = parser.parse_args()
if args.verbose:
log.setLevel(logging.INFO)
if args.quiet:
log.setLevel(logging.CRITICAL)
km = BlockingKernelManager()
km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w'))
km.start_channels()
if args.saveonerror:
# XXX: hack - just dange an attribute we can check against
km.saveonerror = True
halt_on_error = not args.all
halt_notebooks = not args.allnotebooks
for ipynb in args.inputs:
log.info('Running '+ ipynb)
with open(ipynb) as f:
nb = reads(f.read(), 'json')
# here, before running the notebook, it'd be nice if we verified it's
# metadata.name, and adjusted it according to the current filename, so
# let's do that
def km_from_cfg(cfg):
km = BlockingKernelManager(**cfg)
km.start_channels()
km.shell_channel.session.key = km.key
km.hb_channel.unpause()
return km