def test_cleanup_context(self):
km = KernelManager()
self.assertIsNotNone(km.context)
km.cleanup_resources(restart=False)
self.assertTrue(km.context.closed)
def km_from_string(s=''):
"""
Create kernel manager.
"""
global km, kc, send
s = s.replace('--existing', '')
try:
if '--profile' in s:
k, p = s.split('--profile')
k = k.lstrip().rstrip() # kernel part of the string
p = p.lstrip().rstrip() # profile part of the string
fullpath = find_connection_file(k, p)
elif s == '':
fullpath = find_connection_file()
else:
fullpath = find_connection_file(s.lstrip().rstrip())
except IOError:
vim_echo(":IPython " + s + " failed", "Info")
vim_echo("^-- failed '" + s + "' not found", "Error")
return
km = KernelManager(connection_file=fullpath)
km.load_connection_file()
kc = km.client()
kc.start_channels()
send = kc.execute
vim_echo('Connected to {}'.format(fullpath))
def temporary_connect_query(connection_file, code, retvar):
# connect
from jupyter_client import KernelManager
_km = KernelManager(connection_file=connection_file)
_km.load_connection_file()
_kc = _km.client()
_kc.start_channels()
def _send(msg, **kwargs):
"""Send a message to the kernel client."""
# Include dedent of msg so we don't get odd indentation errors.
return _kc.execute(textwrap.dedent(msg), **kwargs)
# send code
msg_id = _send(code, silent=True, user_expressions={"_ret": retvar})
try:
reply = get_reply_msg(msg_id, _kc=_kc)
except Empty:
vim_echom("no reply from jupyter kernel", "WarningMsg")
return -1
# get result
retval = None
try:
# Requires the fix for https://github.com/JuliaLang/IJulia.jl/issues/815
retval = reply["content"]["user_expressions"]["_ret"]["data"][
"text/plain"]
except KeyError:
vim_echom("Could not get return value, kernel not ready?")
# disconnect
_kc.stop_channels()
return retval
def _run_signaltest_lifecycle(self, config=None):
km = KernelManager(config=config, kernel_name="signaltest")
kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE)
def execute(cmd):
request_id = kc.execute(cmd)
while True:
reply = kc.get_shell_msg(TIMEOUT)
if reply["parent_header"]["msg_id"] == request_id:
break
content = reply["content"]
assert content["status"] == "ok"
return content
execute("start")
assert km.is_alive()
execute("check")
assert km.is_alive()
km.restart_kernel(now=True)
assert km.is_alive()
execute("check")
km.shutdown_kernel()
assert km.context.closed
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
shell = kc.shell_channel
# simple ping:
kc.execute("pass")
shell.get_msg()
failures = 0
for cell in nb.cells:
if cell.cell_type != 'code':
continue
kc.execute(cell.source)
try:
# wait for finish, w/ timeout
reply = shell.get_msg(timeout=TIMEOUT)['content']
except Empty:
raise Exception(
'Timeout (%.1f) when executing the following %s cell: "%s"' %
(TIMEOUT, cell.cell_type, cell.source.strip()))
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:", file=sys.stderr)
print('\n'.join(reply['traceback']), file=sys.stderr)
print(file=sys.stderr)
kc.stop_channels()
km.shutdown_kernel()
del km
if failures > 0:
raise Exception()
def _run_signaltest_lifecycle(self, config=None):
km = KernelManager(config=config, kernel_name='signaltest')
kc = self._prepare_kernel(km, stdout=PIPE, stderr=PIPE)
def execute(cmd):
request_id = kc.execute(cmd)
while True:
reply = kc.get_shell_msg(TIMEOUT)
if reply['parent_header']['msg_id'] == request_id:
break
content = reply['content']
assert content['status'] == 'ok'
return content
execute("start")
assert km.is_alive()
execute('check')
assert km.is_alive()
km.restart_kernel(now=True)
assert km.is_alive()
execute('check')
km.shutdown_kernel()
assert km.context.closed
def create_kernel_manager(self):
"""Create the kernel manager and connect a client.
Returns
-------
bool
True if client connects successfully, False on failure.
"""
# Get client
kernel_manager = KernelManager(connection_file=self.cfile)
# The json may be badly encoding especially if autoconnecting
try:
kernel_manager.load_connection_file()
# pylint: disable=broad-except
except Exception:
return False
self.km_client = kernel_manager.client()
# Open channel
self.km_client.start_channels()
# Ping the kernel
self.km_client.kernel_info()
try:
self.km_client.get_shell_msg(timeout=1)
return True
except Empty:
return False
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
shell = kc.shell_channel
# simple ping:
kc.execute("pass")
shell.get_msg()
failures = 0
for cell in nb.cells:
if cell.cell_type != 'code':
continue
kc.execute(cell.source)
# wait for finish, maximum 20s
reply = shell.get_msg(timeout=20)['content']
if reply['status'] == 'error':
failures += 1
print("\nFAILURE:")
print('\n'.join(reply['traceback']))
print()
kc.stop_channels()
km.shutdown_kernel()
del km
if failures > 0:
raise Exception()
def __init__(self,
timeout: int = 100,
template_type: TemplateType = TemplateType.FULL):
"""
Initializes Exporter with default timeout (100 seconds) and template
(FULL) and handles instantiation of km and ep variables for usage when
generating NotebookNodes and their outputs.
Args:
timeout (int): Amount of time in seconds that a visualization can
run for before being stopped.
template_type (TemplateType): Type of template to use when
generating visualization output.
"""
self.timeout = timeout
self.template_type = template_type
# Create custom KernelManager.
# This will circumvent issues where kernel is shutdown after
# preprocessing. Due to the shutdown, latency would be introduced
# because a kernel must be started per visualization.
self.km = KernelManager()
self.km.start_kernel()
self.ep = ExecutePreprocessor(timeout=self.timeout,
kernel_name='python3',
allow_errors=True)
def __init__(self, use_exist=False):
"""
## Description
Initializes the `kernel_manager` and `client` objects
and starts the kernel. Also initializes the pretty printer
for displaying object properties and execution result
payloads.
## Parameters
None.
"""
if not use_exist:
# Initialize kernel and client
self.kernel_manager, self.client = start_new_kernel()
self.send = self.client.execute
else:
self.kernel_manager = KernelManager(
connection_file=find_connection_file())
self.kernel_manager.load_connection_file(find_connection_file())
self.client = self.kernel_manager.client()
self.client.start_channels()
self.send = self.client.execute
# Initialize pretty printer
self.pp = PrettyPrinter(indent=2)
def _check_ipynb():
kernel_manager = KernelManager()
kernel_manager.start_kernel()
kernel_client = kernel_manager.client()
kernel_client.start_channels()
try:
# IPython 3.x
kernel_client.wait_for_ready()
iopub = kernel_client
shell = kernel_client
except AttributeError:
# Ipython 2.x
# Based on https://github.com/paulgb/runipy/pull/49/files
iopub = kernel_client.iopub_channel
shell = kernel_client.shell_channel
shell.get_shell_msg = shell.get_msg
iopub.get_iopub_msg = iopub.get_msg
successes = 0
failures = 0
errors = 0
report = ''
_execute_code("print('Hello World')", shell, iopub, timeout=1)
kernel_client.stop_channels()
kernel_manager.shutdown_kernel()
passed = not (failures or errors)
print(report)
def kernel_client(config):
try:
if config.kernel != 'existing':
cnx_file = find_connection_file(filename=config.kernel)
else:
cnx_file = find_connection_file()
km = KernelManager(connection_file=cnx_file)
km.load_connection_file()
kc = km.client()
kc.start_channels()
try:
msg = kc.shell_channel.get_msg()
except Empty:
try:
# try again :)
sleep(1)
msg = kc.shell_channel.get_msg()
except Empty:
raise RuntimeError(
'No message found in the channel. Is the kernel alive?')
except Exception as e:
print(format_text('PyExecError: ipython_exec was not able to connect to the desired jupyter kernel\n' + \
"Hint: execute 'ipython_start_kernel' or 'jupyter console' first\n" + \
format_exc(), config.to))
exit(1)
return kc
def __init__(
self, parsed, kernel, source, mode, figdir, outdir, embed_kernel=False
):
super(JupyterProcessor, self).__init__(
parsed, kernel, source, mode, figdir, outdir
)
self.extra_arguments = None
self.timeout = -1
path = os.path.abspath(outdir)
if embed_kernel:
km = InProcessKernelManager(kernel_name=kernel)
else:
km = KernelManager(kernel_name=kernel)
km.start_kernel(cwd=path, stderr=open(os.devnull, "w"))
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready()
except RuntimeError:
print(
"Timeout from starting kernel\nTry restarting python session and running weave again"
)
kc.stop_channels()
km.shutdown_kernel()
raise
self.km = km
self.kc = kc
self.kc.allow_stdin = False
def run_notebook(nb):
km = KernelManager()
km.start_kernel(stderr=open(os.devnull, 'w'))
kc = km.client()
kc.start_channels()
# simple ping:
kc.execute("pass")
kc.get_shell_msg()
cells = 0
failures = 0
for cell in nb.cells:
if cell.cell_type != 'code':
continue
outputs, failed = run_cell(kc, cell)
cell.outputs = outputs
cells += 1
cell['execution_count'] = cells
failures += failed
sys.stdout.write('.')
sys.stdout.flush()
print()
#print("ran notebook %s" % nb.metadata.name)
print(" ran %3i cells" % cells)
if failures:
print(" %3i cells raised exceptions" % failures)
kc.stop_channels()
km.shutdown_kernel()
del km
def connect_kernel_from_file(kernel_type, connection_file):
from jupyter_client import KernelManager
# Create the kernel manager and connect a client
# See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
km = KernelManager(connection_file=connection_file)
km.load_connection_file()
kc = km.client()
kc.start_channels()
# Alias execute function
def _send(msg, **kwargs):
"""Send a message to the kernel client."""
# Include dedent of msg so we don't get odd indentation errors.
return kc.execute(textwrap.dedent(msg), **kwargs)
send = _send
# Ping the kernel
kc.kernel_info()
try:
reply = kc.get_shell_msg(timeout=1)
except Empty:
return {"connected": False}
else:
# pid = get_pid(kernel_type) # Ask kernel for its PID
return {"connected": True, "kc": kc, "send": send}
def new_kernel():
"""
Create a new IPython kernel (optionally with extra arguments)
"""
km = KernelManager()
km.start_kernel()
return km_from_string(km.connection_file)
def __init__(self):
"""
Construct an IPythonConnection object without connecting
"""
self.kmgr = KernelManager(kernel_name='python3')
self.client = None
self._status_ = 'disconnected'
self._kernel_ = {}
def start(self):
self.km = KernelManager(kernel_name='python',
client_class='execute.Client')
self.km.start_kernel()
self.kc = self.km.client()
self.kc.start_channels()
time.sleep(2)
atexit.register(self.shutdown_kernel)
def test_notebook(nb):
km = KernelManager()
# Do not save the history to disk, as it can yield spurious lock errors.
# See https://github.com/ipython/ipython/issues/2845
km.start_kernel(extra_arguments=['--HistoryManager.hist_file=:memory:'])
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready()
except AttributeError:
_wait_for_ready_backport(kc)
successes = 0
failures = 0
errors = 0
for ws in nb.worksheets:
for i, cell in enumerate(ws.cells):
if cell.cell_type != 'code' or cell.input.startswith('%timeit'):
continue
try:
outs = run_cell(kc, cell)
except Exception as e:
print("failed to run cell:", repr(e))
print(cell.input)
errors += 1
continue
failed = False
if len(outs) != len(cell.outputs):
print("output length mismatch (expected {}, got {})".format(
len(cell.outputs), len(outs)))
failed = True
for out, ref in zip(outs, cell.outputs):
if not compare_outputs(out, ref):
failed = True
print("cell %d: " % i, end="")
if failed:
print("FAIL")
failures += 1
else:
print("OK")
successes += 1
print()
print("tested notebook %s" % nb.metadata.name)
print(" %3i cells successfully replicated" % successes)
if failures:
print(" %3i cells mismatched output" % failures)
if errors:
print(" %3i cells failed to complete" % errors)
kc.stop_channels()
km.shutdown_kernel()
del km
if failures | errors:
sys.exit(1)
def test_no_cleanup_shared_context(self, zmq_context):
"""kernel manager does not terminate shared context"""
km = KernelManager(context=zmq_context)
assert km.context == zmq_context
assert km.context is not None
km.cleanup_resources(restart=False)
assert km.context.closed is False
assert zmq_context.closed is False
def __init__(self, *args, **kwargs):
super(FormatManager, self).__init__(*args, **kwargs)
if self._instance is not None:
raise ValueError("FormatManager is a singleton, access with"
" FormatManager.format_manager")
self._formats = {}
self._km = KernelManager()
def connect_to_kernel(self):
"""Create kernel manager from existing connection file."""
from jupyter_client import KernelManager, find_connection_file
# Test if connection is alive
connected = self.check_connection()
attempt = 0
max_attempts = 3
while not connected and attempt < max_attempts:
attempt += 1
try:
cfile = find_connection_file(
) # default filename='kernel-*.json'
except IOError:
self.vim_echom(
"kernel connection attempt {:d} failed - no kernel file".
format(attempt),
style="Error")
continue
# Create the kernel manager and connect a client
# See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
km = KernelManager(connection_file=cfile)
km.load_connection_file()
self.kc = km.client()
self.kc.start_channels()
# Alias execute function
def _send(msg, **kwargs):
"""Send a message to the kernel client."""
# Include dedent of msg so we don't get odd indentation errors.
return self.kc.execute(textwrap.dedent(msg), **kwargs)
self.send = _send
# Ping the kernel
self.kc.kernel_info()
try:
self.reply = self.kc.get_shell_msg(timeout=1)
except Empty:
continue
else:
connected = True
if connected:
# Send command so that monitor knows vim is commected
# send('"_vim_client"', store_history=False)
self.pid = self.set_pid() # Ask kernel for its PID
self.vim_echom('kernel connection successful! pid = {}'.format(
self.pid),
style='Question')
else:
self.kc.stop_channels()
self.vim_echom('kernel connection attempt timed out',
style='Error')
def start(self, cwd=None, executable=None):
self.km = KernelManager(kernel_name='python',
client_class='nei.execute.Client')
self.executable = executable
if executable is not None:
self.km.kernel_spec.argv[0] = executable
self.km.start_kernel(**({} if cwd is None else {'cwd': cwd}))
self.kc = self.km.client()
self.kc.start_channels()
time.sleep(2)
atexit.register(self.shutdown_kernel)
def kernel_manager(self, os_safe_transport):
"""Prepare a KernelManager for the given transports.
Adapted from:
- https://github.com/jupyter/jupyter_client/blob/284914b/jupyter_client/tests/test_kernelmanager.py#L47
- https://github.com/jupyter/jupyter_client/blob/284914b/jupyter_client/tests/test_kernelmanager.py#L79
"""
km_config = Config()
km_config.KernelManager.transport = os_safe_transport
if os_safe_transport == "ipc":
km_config.KernelManager.ip = "test"
return KernelManager(config=km_config)
def _block_and_receive_results(*return_values, save_output_notebook=None):
# add an extra cell to beginning of notebook to populate parameters
notebook_parameters = __notebookscripter_injected__ + [[hooks, {"return_values": return_values}]]
base64_parameters = obj_to_string_literal(notebook_parameters)
initialization_source = """from NotebookScripter import (rehydrate as __rehydrate__, dehydrate_return_values as __dehydrate_return_values__)
__rehydrate__({})""".format(base64_parameters)
initialization_cell = notebook_node_from_dict({
"cell_type": "code",
"execution_count": 0,
"metadata": {},
"outputs": [],
"source": initialization_source
})
finalization_source = """__dehydrate_return_values__(locals())"""
finalization_cell = notebook_node_from_dict({
"cell_type": "code",
"execution_count": 0,
"metadata": {},
"outputs": [],
"source": finalization_source})
notebook['cells'].insert(0, initialization_cell)
notebook['cells'].append(finalization_cell)
km = KernelManager()
# hack -- needed because the code within ExecutePreprocessor.start_kernel to start
# the kernel when km hasn't started a kernel already can't possibly work
km.start_kernel()
executed_notebook = executenb(notebook, timeout=None, km=km)
km.shutdown_kernel()
if save_output_notebook:
if isinstance(save_output_notebook, str):
with open(save_output_notebook, 'w') as f:
write_notebook(executed_notebook, f)
else:
write_notebook(executed_notebook, save_output_notebook)
encoded_return_values = eval(executed_notebook["cells"][-1]["outputs"][0]["data"]["text/plain"])
final_namespace = str_to_obj(encoded_return_values)
module_identity = "loaded_notebook_from_subprocess"
dynamic_module = types.ModuleType(module_identity)
dynamic_module.__file__ = path_to_notebook
# inject retrieved return values into the returned module namespace
dynamic_module.__dict__.update(final_namespace)
return dynamic_module
def setUp(self):
try:
KernelSpecManager().get_kernel_spec(NATIVE_KERNEL_NAME)
except NoSuchKernel:
pytest.skip()
self.km = KernelManager(kernel_name=NATIVE_KERNEL_NAME)
self.km.start_kernel()
self.kc = self.km.client()
self.kc.start_channels()
self.addCleanup(self.kc.stop_channels)
self.addCleanup(self.km.shutdown_kernel)
def setUp(self):
try:
KernelSpecManager().get_kernel_spec('python3_clean')
except NoSuchKernel:
pytest.skip()
self.km = KernelManager(kernel_name='python3_clean')
self.km.start_kernel()
self.kc = self.km.client()
self.kc.start_channels()
self.addCleanup(self.kc.stop_channels)
self.addCleanup(self.km.shutdown_kernel)
def test_notebook(ipynb):
with open(ipynb, encoding='utf-8') as f:
nb = nbformat.reads_json(f.read())
km = KernelManager()
# Do not save the history to disk, as it can yield spurious lock errors.
# See https://github.com/ipython/ipython/issues/2845
km.start_kernel(extra_arguments=['--HistoryManager.hist_file=:memory:'])
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready(timeout=30)
except AttributeError:
_wait_for_ready_backport(kc)
successes = 0
failures = 0
errors = 0
for i, cell in enumerate(nb.cells):
if cell.cell_type != 'code' or cell.source.startswith('%timeit'):
continue
try:
outs = run_cell(kc, cell)
except Exception as e:
print("failed to run cell:", repr(e))
print(cell.input)
errors += 1
continue
failed = False
if not compare_outputs(cell.outputs, outs):
failed = True
print("cell %d: " % i, end="")
if failed:
print("FAIL")
failures += 1
else:
print("OK")
successes += 1
print("tested notebook %s" % ipynb)
print(" %3i cells successfully replicated" % successes)
if failures:
print(" %3i cells mismatched output" % failures)
if errors:
print(" %3i cells failed to complete" % errors)
kc.stop_channels()
km.shutdown_kernel()
del km
if failures | errors:
sys.exit(1)
def start_kernel():
manager = KernelManager()
manager.start_kernel()
port_names = ["shell", "stdin", "iopub", "hb", "control"]
ports = dict(list(zip(port_names, manager.ports)))
kernel_info = {
"key": manager.session.key.decode("utf-8"),
"ports": ports,
}
return json.dumps(kernel_info)
def test_no_cleanup_shared_context(self):
"""kernel manager does not terminate shared context"""
import zmq
ctx = zmq.Context()
km = KernelManager(context=ctx)
self.assertEquals(km.context, ctx)
self.assertIsNotNone(km.context)
km.cleanup_resources(restart=False)
self.assertFalse(km.context.closed)
self.assertFalse(ctx.closed)
ctx.term()
