def setup_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
kernel = Popen([sys.executable, "-c", cmd], stdout=PIPE, stderr=PIPE, env=env)
connection_file = os.path.join(IPYTHONDIR, "profile_default", "security", "kernel-%i.json" % kernel.pid)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + SETUP_TIMEOUT:
time.sleep(0.1)
if kernel.poll() is not None:
o, e = kernel.communicate()
e = py3compat.cast_unicode(e)
raise IOError("Kernel failed to start:\n%s" % e)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise IOError("Connection file %r never arrived" % connection_file)
client = BlockingKernelClient(connection_file=connection_file)
client.load_connection_file()
client.start_channels()
client.wait_for_ready()
try:
yield client
finally:
client.stop_channels()
kernel.terminate()
def setup_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
def connection_file_ready(connection_file):
"""Check if connection_file is a readable json file."""
if not os.path.exists(connection_file):
return False
try:
with open(connection_file) as f:
json.load(f)
return True
except ValueError:
return False
kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE)
try:
connection_file = os.path.join(
paths.jupyter_runtime_dir(),
'kernel-%i.json' % kernel.pid,
)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while not connection_file_ready(connection_file) \
and kernel.poll() is None \
and time.time() < tic + SETUP_TIMEOUT:
time.sleep(0.1)
# Wait 100ms for the writing to finish
time.sleep(0.1)
if kernel.poll() is not None:
o,e = kernel.communicate()
e = py3compat.cast_unicode(e)
raise IOError("Kernel failed to start:\n%s" % e)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise IOError("Connection file %r never arrived" % connection_file)
client = BlockingKernelClient(connection_file=connection_file)
client.load_connection_file()
client.start_channels()
client.wait_for_ready()
try:
yield client
finally:
client.stop_channels()
finally:
kernel.terminate()
def test_start_ipython_workers(loop, zmq_ctx):
from jupyter_client import BlockingKernelClient
with cluster(1) as (s, [a]):
with Client(s["address"], loop=loop) as e:
info_dict = e.start_ipython_workers()
info = first(info_dict.values())
kc = BlockingKernelClient()
kc.load_connection_info(info)
kc.start_channels()
kc.wait_for_ready(timeout=10)
msg_id = kc.execute("worker")
reply = kc.get_shell_msg(timeout=10)
assert reply["parent_header"]["msg_id"] == msg_id
assert reply["content"]["status"] == "ok"
kc.stop_channels()
def test_start_ipython_workers(loop, zmq_ctx):
from jupyter_client import BlockingKernelClient
with cluster(1) as (s, [a]):
with Executor(('127.0.0.1', s['port']), loop=loop) as e:
info_dict = e.start_ipython_workers()
info = first(info_dict.values())
key = info.pop('key')
kc = BlockingKernelClient(**info)
kc.session.key = key
kc.start_channels()
kc.wait_for_ready(timeout=10)
msg_id = kc.execute("worker")
reply = kc.get_shell_msg(timeout=10)
assert reply['parent_header']['msg_id'] == msg_id
assert reply['content']['status'] == 'ok'
kc.stop_channels()
def test_start_ipython_workers(loop, zmq_ctx):
from jupyter_client import BlockingKernelClient
with cluster(1) as (s, [a]):
with Client(('127.0.0.1', s['port']), loop=loop) as e:
info_dict = e.start_ipython_workers()
info = first(info_dict.values())
key = info.pop('key')
kc = BlockingKernelClient(**info)
kc.session.key = key
kc.start_channels()
kc.wait_for_ready(timeout=10)
msg_id = kc.execute("worker")
reply = kc.get_shell_msg(timeout=10)
assert reply['parent_header']['msg_id'] == msg_id
assert reply['content']['status'] == 'ok'
kc.stop_channels()
def setup_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
This function was taken from the ipykernel project.
We plan to remove it when dropping support for python 2.
Yields
-------
client: jupyter_client.BlockingKernelClient connected to the kernel
"""
kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE)
try:
connection_file = os.path.join(
paths.jupyter_runtime_dir(),
'kernel-%i.json' % kernel.pid,
)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while not os.path.exists(connection_file) \
and kernel.poll() is None \
and time.time() < tic + SETUP_TIMEOUT:
time.sleep(0.1)
if kernel.poll() is not None:
o, e = kernel.communicate()
if not PY3 and isinstance(e, bytes):
e = e.decode()
raise IOError("Kernel failed to start:\n%s" % e)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise IOError("Connection file %r never arrived" % connection_file)
client = BlockingKernelClient(connection_file=connection_file)
client.load_connection_file()
client.start_channels()
client.wait_for_ready()
try:
yield client
finally:
client.stop_channels()
finally:
if not PY2:
kernel.terminate()
def setup():
global client
kernel = Popen([sys.executable, '-m', 'ipykernel'], stdout=PIPE, stderr=PIPE)
connection_file = os.path.join(
paths.jupyter_runtime_dir(),
'kernel-%i.json' % kernel.pid,
)
sleep(1)
client = BlockingKernelClient(connection_file=connection_file)
client.load_connection_file()
client.start_channels()
client.wait_for_ready()
loaded = client.execute_interactive(load_splonky)
if loaded['content']['status'] == 'error':
raise Exception("Could not load core Splonky libraries")
os_process_id = re.findall('.*\/kernel-(\d+)\.json$', connection_file)[0]
return os_process_id
def remote_magic(line, cell=None):
"""A magic for running code on a specified remote worker
The connection_info dict of the worker will be looked up
as the first positional arg to the magic.
The rest of the line (or the entire cell for a %%cell magic)
will be passed to the remote kernel.
Usage:
info = e.start_ipython(worker)[worker]
%remote info print(worker.data)
"""
# get connection info from IPython's user namespace
ip = get_ipython()
split_line = line.split(None, 1)
info_name = split_line[0]
if info_name not in ip.user_ns:
raise NameError(info_name)
connection_info = dict(ip.user_ns[info_name])
if not cell: # line magic, use the rest of the line
if len(split_line) == 1:
raise ValueError("I need some code to run!")
cell = split_line[1]
# turn info dict to hashable str for use as lookup key in _clients cache
key = ",".join(map(str, sorted(connection_info.items())))
session_key = connection_info.pop("key")
if key in remote_magic._clients:
kc = remote_magic._clients[key]
else:
kc = BlockingKernelClient(**connection_info)
kc.session.key = session_key
kc.start_channels()
kc.wait_for_ready(timeout=10)
remote_magic._clients[key] = kc
# actually run the code
run_cell_remote(ip, kc, cell)
def remote_magic(line, cell=None):
"""A magic for running code on a specified remote worker
The connection_info dict of the worker will be looked up
as the first positional arg to the magic.
The rest of the line (or the entire cell for a %%cell magic)
will be passed to the remote kernel.
Usage:
info = e.start_ipython(worker)[worker]
%remote info print(worker.data)
"""
# get connection info from IPython's user namespace
ip = get_ipython()
split_line = line.split(None, 1)
info_name = split_line[0]
if info_name not in ip.user_ns:
raise NameError(info_name)
connection_info = dict(ip.user_ns[info_name])
if not cell: # line magic, use the rest of the line
if len(split_line) == 1:
raise ValueError("I need some code to run!")
cell = split_line[1]
# turn info dict to hashable str for use as lookup key in _clients cache
key = ','.join(map(str, sorted(connection_info.items())))
session_key = connection_info.pop('key')
if key in remote_magic._clients:
kc = remote_magic._clients[key]
else:
kc = BlockingKernelClient(**connection_info)
kc.session.key = session_key
kc.start_channels()
kc.wait_for_ready(timeout=10)
remote_magic._clients[key] = kc
# actually run the code
run_cell_remote(ip, kc, cell)
def setup_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
kernel = Popen([sys.executable, '-c', cmd],
stdout=PIPE,
stderr=PIPE,
env=env)
connection_file = os.path.join(IPYTHONDIR, 'profile_default', 'security',
'kernel-%i.json' % kernel.pid)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while not os.path.exists(connection_file) \
and kernel.poll() is None \
and time.time() < tic + SETUP_TIMEOUT:
time.sleep(0.1)
if kernel.poll() is not None:
o, e = kernel.communicate()
e = py3compat.cast_unicode(e)
raise IOError("Kernel failed to start:\n%s" % e)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise IOError("Connection file %r never arrived" % connection_file)
client = BlockingKernelClient(connection_file=connection_file)
client.load_connection_file()
client.start_channels()
client.wait_for_ready()
try:
yield client
finally:
client.stop_channels()
kernel.terminate()
def setup_kernel(cmd):
"""start an embedded kernel in a subprocess, and wait for it to be ready
Returns
-------
kernel_manager: connected KernelManager instance
"""
def connection_file_ready(connection_file):
"""Check if connection_file is a readable json file."""
if not os.path.exists(connection_file):
return False
try:
with open(connection_file) as f:
json.load(f)
return True
except ValueError:
return False
kernel = Popen([sys.executable, "-c", cmd], stdout=PIPE, stderr=PIPE, encoding="utf-8")
try:
connection_file = os.path.join(
paths.jupyter_runtime_dir(),
"kernel-%i.json" % kernel.pid,
)
# wait for connection file to exist, timeout after 5s
tic = time.time()
while (
not connection_file_ready(connection_file)
and kernel.poll() is None
and time.time() < tic + SETUP_TIMEOUT
):
time.sleep(0.1)
# Wait 100ms for the writing to finish
time.sleep(0.1)
if kernel.poll() is not None:
o, e = kernel.communicate()
raise OSError("Kernel failed to start:\n%s" % e)
if not os.path.exists(connection_file):
if kernel.poll() is None:
kernel.terminate()
raise OSError("Connection file %r never arrived" % connection_file)
client = BlockingKernelClient(connection_file=connection_file)
client.load_connection_file()
client.start_channels()
client.wait_for_ready()
try:
yield client
finally:
client.stop_channels()
finally:
kernel.terminate()
kernel.wait()
# Make sure all the fds get closed.
for attr in ["stdout", "stderr", "stdin"]:
fid = getattr(kernel, attr)
if fid:
fid.close()