def kernel_client(config):
try:
if config.kernel != 'existing':
cnx_file = find_connection_file(filename=config.kernel)
else:
cnx_file = find_connection_file()
km = KernelManager(connection_file=cnx_file)
km.load_connection_file()
kc = km.client()
kc.start_channels()
try:
msg = kc.shell_channel.get_msg()
except Empty:
try:
# try again :)
sleep(1)
msg = kc.shell_channel.get_msg()
except Empty:
raise RuntimeError(
'No message found in the channel. Is the kernel alive?')
except Exception as e:
print(format_text('PyExecError: ipython_exec was not able to connect to the desired jupyter kernel\n' + \
"Hint: execute 'ipython_start_kernel' or 'jupyter console' first\n" + \
format_exc(), config.to))
exit(1)
return kc
def km_from_string(s=''):
"""
Create kernel manager.
"""
global km, kc, send
s = s.replace('--existing', '')
try:
if '--profile' in s:
k, p = s.split('--profile')
k = k.lstrip().rstrip() # kernel part of the string
p = p.lstrip().rstrip() # profile part of the string
fullpath = find_connection_file(k, p)
elif s == '':
fullpath = find_connection_file()
else:
fullpath = find_connection_file(s.lstrip().rstrip())
except IOError:
vim_echo(":IPython " + s + " failed", "Info")
vim_echo("^-- failed '" + s + "' not found", "Error")
return
km = KernelManager(connection_file=fullpath)
km.load_connection_file()
kc = km.client()
kc.start_channels()
send = kc.execute
vim_echo('Connected to {}'.format(fullpath))
def __init__(self, use_exist=False):
"""
## Description
Initializes the `kernel_manager` and `client` objects
and starts the kernel. Also initializes the pretty printer
for displaying object properties and execution result
payloads.
## Parameters
None.
"""
if not use_exist:
# Initialize kernel and client
self.kernel_manager, self.client = start_new_kernel()
self.send = self.client.execute
else:
self.kernel_manager = KernelManager(
connection_file=find_connection_file())
self.kernel_manager.load_connection_file(find_connection_file())
self.client = self.kernel_manager.client()
self.client.start_channels()
self.send = self.client.execute
# Initialize pretty printer
self.pp = PrettyPrinter(indent=2)
def start_action(args, lockfile, Config):
""" Start Parser action. """
if args.integer:
try:
kid = str(args.integer)
find_connection_file(kid)
except OSError:
message = '{}Error :\t{}Cannot find kernel id. {} !\n\tExiting\n'
sys.stderr.write(message.format(RED, RESET, args.integer))
sys.exit(1)
else:
message = 'Connecting to kernel id. {}\n'
sys.stdout.write(message.format(kid))
elif args.action == 'last':
kid = kdread(lockfile)
if kid:
message = 'Connecting to kernel id. {}\n'
sys.stdout.write(message.format(kid))
else:
sys.exit(1)
else:
kid = create_new(Config)
kdwrite(lockfile, kid)
return kid
def parse_args(lockfile, pidfile):
""" Parse Arguments. """
parser = argparse.ArgumentParser()
parser.add_argument("-L", "--list", help="List all kernels",
action="store_true")
parser.add_argument("integer", help="Start up with existing kernel. \
INTEGER is the id of the connection file. \
INTEGER can also be the keyword 'last' for 'last kernel'",
nargs='?')
args = parser.parse_args()
pid = kd_status(pidfile)
if args.list:
print_kernel_list()
sys.exit(0)
elif os.path.exists(lockfile) and pid:
try:
cf = init_cf(lockfile)
except OSError:
sys.stderr.write('lockfile points to an unknown connection file.\n')
sys.stderr.write("Try 'kd5 stop'\n")
sys.exit(1)
if args.integer:
message = 'Daemon is already running. Dropping argument {}\n'
sys.stderr.write(message.format(args.integer))
time.sleep(1.5)
elif not os.path.exists(lockfile) and pid:
no_lock_exit()
elif args.integer == 'last' and not os.path.exists(lockfile):
no_lock_exit()
elif args.integer == 'last' and os.path.exists(lockfile):
cmd = 'kd5 last'
cf = with_daemon(lockfile, pidfile, cmd)
elif args.integer:
try:
find_connection_file(str(args.integer))
except OSError:
message = '{}Error :{}\tCannot find kernel id. {} !\n\tExiting\n'
sys.stderr.write(message.format(RED, RESET, args.integer))
sys.exit(1)
else:
cmd = 'kd5 start ' + str(args.integer)
cf = with_daemon(lockfile, pidfile, cmd)
else:
cmd = 'kd5 start'
cf = with_daemon(lockfile, pidfile, cmd)
return args, cf
def main(args=None):
""" Main """
cfg = cfg_setup()
Config = cfg.run()
logdir = os.path.expanduser('~') + '/.cpyvke/'
logfile = logdir + 'kd5.log'
lockfile = logdir + 'kd5.lock'
pidfile = logdir + 'kd5.pid'
# Logger
logger.setLevel(logging.DEBUG)
# create the logging file handler
handler = RotatingFileHandler(logfile,
maxBytes=10 * 1024 * 1024,
backupCount=5)
logmsg = '%(asctime)s :: %(name)s :: %(threadName)s :: %(levelname)s :: %(message)s'
formatter = logging.Formatter(logmsg, datefmt='%Y-%m-%d - %H:%M:%S')
handler.setFormatter(formatter)
logger.addHandler(handler)
# Parse Arguments
args, kid = parse_args(lockfile, pidfile, Config)
sport = int(Config['comm']['s-port'])
rport = int(Config['comm']['r-port'])
delay = float(Config['daemon']['refresh'])
try:
cfile = find_connection_file(kid)
except OSError:
sys.stderr.write('Lockfile points to an unknown connection file !\n')
kid = create_new(Config)
cfile = find_connection_file(kid)
kdwrite(lockfile, kid)
WatchConf = {'cf': cfile, 'delay': delay, 'sport': sport, 'rport': rport}
daemon = Daemonize(pidfile, WatchConf, stdout=logfile, stderr=logfile)
if args.action == 'stop':
daemon.stop()
sys.stdout.write('kd5 stopped !\n')
elif args.action == 'start' or args.action == 'last':
sys.stdout.write('kd5 starting...\n')
daemon.start()
elif args.action == 'restart':
sys.stdout.write('kd5 restarting...\n')
daemon.restart()
def running_kernels(self, args):
rdir = jupyter_runtime_dir()
l = fnmatch.filter(os.listdir(rdir), 'kernel-*.json')
if len(l) > 1:
cf = os.path.relpath(find_connection_file(), rdir)
l = [f + '(newest)' if f == cf else f for f in l]
return l
def main(kid):
# Load connection info and init communications.
cf = find_connection_file(kid)
km = BlockingKernelClient(connection_file=cf)
km.load_connection_file()
km.start_channels()
# Define a function that is useful from within the user's notebook: juneau_connect() can be
# used to directly connect the notebook to the source database. Note that this includes the
# full "root" credentials.
# FIXME: allow for user-specific credentials on SQL tables. The DBMS may also not be at localhost.
code = f"""
from sqlalchemy import create_engine
def juneau_connect():
engine = create_engine(
"postgresql://{config.sql.name}:{config.sql.password}@{config.sql.host}/{config.sql.dbname}",
connect_args={{
"options": "-csearch_path='{config.sql.dbs}'"
}}
)
return engine.connect()
"""
km.execute_interactive(code, timeout=TIMEOUT)
km.stop_channels()
def _createConsoleWidget(self):
if is_using_pyqt5():
layout = QtWidgets.QVBoxLayout()
else:
layout = QtGui.QVBoxLayout()
connection_file = find_connection_file(self.connection_file)
self.kernel_manager = QtKernelManager(connection_file=connection_file)
self.kernel_manager.load_connection_file()
self.kernel_manager.client_factory = QtKernelClient
self.kernel_client = self.kernel_manager.client()
self.kernel_client.start_channels()
widget_options = {}
if sys.platform.startswith('linux'):
# Some upstream bug crashes IDA when the ncurses completion is
# used. I'm not sure where the bug is exactly (IDA's Qt5 bindings?)
# but using the "droplist" instead works around the crash. The
# problem is present only on Linux.
# See: https://github.com/eset/ipyida/issues/8
widget_options["gui_completion"] = 'droplist'
self.ipython_widget = IdaRichJupyterWidget(self.parent, **widget_options)
self.ipython_widget.kernel_manager = self.kernel_manager
self.ipython_widget.kernel_client = self.kernel_client
layout.addWidget(self.ipython_widget)
return layout
def _createConsoleWidget(self):
if is_using_pyqt5():
layout = QtWidgets.QVBoxLayout()
else:
layout = QtGui.QVBoxLayout()
connection_file = find_connection_file(self.connection_file)
self.kernel_manager = QtKernelManager(connection_file=connection_file)
self.kernel_manager.load_connection_file()
self.kernel_manager.client_factory = QtKernelClient
self.kernel_client = self.kernel_manager.client()
self.kernel_client.start_channels()
widget_options = {}
if sys.platform.startswith('linux'):
# Some upstream bug crashes IDA when the ncurses completion is
# used. I'm not sure where the bug is exactly (IDA's Qt5 bindings?)
# but using the "droplist" instead works around the crash. The
# problem is present only on Linux.
# See: https://github.com/eset/ipyida/issues/8
widget_options["gui_completion"] = 'droplist'
widget_options.update(_user_widget_options)
if ipyida.kernel.is_using_ipykernel_5():
self.ipython_widget = RichJupyterWidget(self.parent,
**widget_options)
else:
self.ipython_widget = IdaRichJupyterWidget(self.parent,
**widget_options)
self.ipython_widget.kernel_manager = self.kernel_manager
self.ipython_widget.kernel_client = self.kernel_client
layout.addWidget(self.ipython_widget)
return layout
def _get_kernel():
# Get Jupyter console kernel info. If there is no console,
# we'll fail here
cf = jupyter_client.find_connection_file()
km = jupyter_client.BlockingKernelClient(connection_file=cf)
km.load_connection_file()
return km
def _get_notebook_name() -> str:
"""
Return the full path of the jupyter notebook.
See https://github.com/jupyter/notebook/issues/1000
"""
try:
connection_file = jupyter_client.find_connection_file()
except OSError:
return ''
kernel_id = re.search('kernel-(.*).json', connection_file).group(1)
servers = _list_running_servers()
for ss in servers:
response = requests.get(urljoin(ss['url'], 'api/sessions'),
params={'token': ss.get('token', '')})
for nn in json.loads(response.text):
if nn['kernel']['id'] == kernel_id:
try:
relative_path = nn['notebook']['path']
return os.path.join(ss['notebook_dir'], relative_path)
except KeyError:
return ''
return ''
def database_name():
sessions = load(
urlopen(
f"{server_info['url']}api/sessions?token={server_info['token']}"))
connection_file = find_connection_file()
session = next(session for session in sessions
if session['kernel']['id'] in connection_file)
with connect(str(root.joinpath('Databases.db'))) as conn:
cursor = conn.cursor()
cursor.execute(
'SELECT name FROM Databases WHERE origin = :origin AND description = :description',
dict(origin=origin, description=session['path']))
result = cursor.fetchone()
if not result:
# There is no web sql database associated with this notebook name so let's create one
# using the session guid. This raises a sqlite3.IntegrityError if you try to insert a
# second table within the same session (UNIQUE constraint failed)
cursor.execute(
'''INSERT INTO Databases (origin, name, description, estimated_size)
VALUES (:origin, :name, :description, :estimated_size)''',
dict(origin=origin,
name=session['id'],
estimated_size=1024 * 1024,
description=session['path']))
result = session['id'],
return result[0]
def init_cf(lockfile):
""" Init connection file. """
with open(lockfile, 'r') as f:
kid = f.readline()
return find_connection_file(kid)
def connect(self, connection_file=None):
debug('connect')
self.has_connection = False
self.km = RedirectingKernelManager(
client_class='jupyter_client.blocking.BlockingKernelClient')
if isinstance(connection_file, str) and not os.path.isfile(
connection_file):
try:
connection_file = find_connection_file(connection_file)
except OSError:
return
try:
self.km.load_connection_file(connection_file=connection_file)
except OSError:
self.vim.out_write('Could not load Jupyter connection file')
return
if self.kc:
self.kc.stop_channels()
self.kc = None
self.kc = self.km.client()
self.kc.shell_channel.call_handlers = self.on_shell_msg
self.kc.hb_channel.call_handlers = self.on_hb_msg
self.kc.start_channels()
if self.waitfor(self.kc.kernel_info()):
self.has_connection = True
def _connect_to_kernel(self, args):
"""Start new or connect to existing `Jupyter` kernel
Parameters
----------
args: argparse.ArgumentParser parsed arguments
Arguments given to `JKernel` command through `neovim`.
Returns
-------
kc: jupyter_client.KernelClient
The kernel client in charge of negotiating communication between
`neovim` and the `Jupyter` kernel.
new_kernel_started: bool
Flag to keep track of new / existing kernel.
"""
l.debug('ARGS {}'.format(args))
if args.existing is not None:
connection_file = jc.find_connection_file(filename=args.existing)
l.debug('CONNECT {}, {}'.format(connection_file, args.existing))
km = jc.KernelManager(connection_file=connection_file)
km.load_connection_file()
new_kernel_started = False
else:
km = jc.KernelManager()
km.start_kernel()
new_kernel_started = True
kc = km.client()
kc.start_channels()
return kc, new_kernel_started
def _connect_to_kernel(self, args):
"""Start new or connect to existing `Jupyter` kernel
Parameters
----------
args: argparse.ArgumentParser parsed arguments
Arguments given to `JKernel` command through `neovim`.
Returns
-------
kc: jupyter_client.KernelClient
The kernel client in charge of negotiating communication between
`neovim` and the `Jupyter` kernel.
new_kernel_started: bool
Flag to keep track of new / existing kernel.
"""
logging.debug('ARGS {}'.format(args))
if args.existing is not None:
connection_file = jc.find_connection_file(filename=args.existing)
logging.debug('CONNECT {}, {}'.format(connection_file,
args.existing))
km = jc.KernelManager(connection_file=connection_file)
km.load_connection_file()
new_kernel_started = False
else:
km = jc.KernelManager()
km.start_kernel()
new_kernel_started = True
kc = km.client()
kc.start_channels()
return kc, new_kernel_started
def exec_code(kid, code):
"""
Executes arbitrary `code` in the kernel with id `kid`.
Returns:
- tuple: the output of the code and the error, if any.
"""
# Load connection info and init communications.
cf = find_connection_file(kid)
with jupyter_lock:
km = BlockingKernelClient(connection_file=cf)
km.load_connection_file()
km.start_channels()
msg_id = km.execute(code, store_history=False)
reply = km.get_shell_msg(msg_id, timeout=60)
output, error = None, None
while km.is_alive():
msg = km.get_iopub_msg(timeout=10)
if ("content" in msg and "name" in msg["content"]
and msg["content"]["name"] == "stdout"):
output = msg["content"]["text"]
break
km.stop_channels()
if reply["content"]["status"] != "ok":
logging.error(f"Status is {reply['content']['status']}")
logging.error(output)
error = output
output = None
return output, error
def exec_code(kid, var, code):
# load connection info and init communication
cf = find_connection_file(kid) # str(port))
global jupyter_lock
jupyter_lock.acquire()
try:
km = BlockingKernelClient(connection_file=cf)
km.load_connection_file()
km.start_channels()
# logging.debug('Executing:\n' + str(code))
msg_id = km.execute(code, store_history=False)
reply = km.get_shell_msg(msg_id, timeout=10)
# logging.info('Execution reply:\n' + str(reply))
state = 'busy'
output = None
idle_count = 0
try:
while km.is_alive():
try:
msg = km.get_iopub_msg(timeout=10)
# logging.debug('Read ' + str(msg))
if not 'content' in msg:
continue
if 'name' in msg['content'] and msg['content'][
'name'] == 'stdout':
# logging.debug('Got data '+ msg['content']['text'])
output = msg['content']['text']
break
if 'execution_state' in msg['content']:
# logging.debug('Got state')
state = msg['content']['execution_state']
if state == 'idle':
idle_count = idle_count + 1
except Empty:
pass
except KeyboardInterrupt:
logging.error('Keyboard interrupt')
pass
finally:
# logging.info('Kernel IO finished')
km.stop_channels()
# logging.info(str(output))
error = ''
if reply['content']['status'] != 'ok':
logging.error('Status is ' + reply['content']['status'])
logging.error(str(output))
error = output
output = None
finally:
jupyter_lock.release()
return output, error
def start(self, metadata: Dict[str, str], database: int = 1) -> None:
"""Method called in a periodically scheduled async worker that should check the dev env and manage Activity
Monitor Instances as needed
Args:
metadata(dict): A dictionary of data to start the activity monitor
database(int): The database ID to use
Returns:
None
"""
# Connect to the kernel
cf = jupyter_client.find_connection_file(
metadata["kernel_id"], path=os.environ['JUPYTER_RUNTIME_DIR'])
km = jupyter_client.BlockingKernelClient()
with open(cf, 'rt') as cf_file:
cf_data = json.load(cf_file)
# Get IP address of lab book container on the bridge network
container_ip = self.get_container_ip()
if not container_ip:
raise ValueError("Failed to find LabBook container IP address.")
cf_data['ip'] = container_ip
km.load_connection_info(cf_data)
# Get connection to the DB
redis_conn = redis.Redis(db=database)
try:
while True:
try:
# Check for messages, waiting up to 1 second. This is the rate that records will be merged
msg = km.get_iopub_msg(timeout=1)
self.handle_message(msg)
except queue.Empty:
# if queue is empty and the record is ready to store, save it!
if self.can_store_activity_record is True:
self.store_record(metadata)
# Check if you should exit
if redis_conn.hget(self.monitor_key,
"run").decode() == "False":
logger.info(
"Received Activity Monitor Shutdown Message for {}".
format(metadata["kernel_id"]))
break
except Exception as err:
logger.error(
"Error in JupyterLab Activity Monitor: {}".format(err))
finally:
# Delete the kernel monitor key so the dev env monitor will spin up a new process
# You may lose some activity if this happens, but the next action will sweep up changes
redis_conn.delete(self.monitor_key)
def create_client(name):
cf = client.find_connection_file('emacs-' + name)
c = client.BlockingKernelClient(connection_file=cf)
c.load_connection_file()
c.start_channels()
chans = [('io', c.get_iopub_msg), ('shell', c.get_shell_msg), ('stdin', c.get_stdin_msg)]
for name, ch in chans:
t = threading.Thread(target=msg_router, args=(name, ch))
t.start()
return c
def connect_to_kernel(self):
"""Create kernel manager from existing connection file."""
from jupyter_client import KernelManager, find_connection_file
# Test if connection is alive
connected = self.check_connection()
attempt = 0
max_attempts = 3
while not connected and attempt < max_attempts:
attempt += 1
try:
cfile = find_connection_file(
) # default filename='kernel-*.json'
except IOError:
self.vim_echom(
"kernel connection attempt {:d} failed - no kernel file".
format(attempt),
style="Error")
continue
# Create the kernel manager and connect a client
# See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
km = KernelManager(connection_file=cfile)
km.load_connection_file()
self.kc = km.client()
self.kc.start_channels()
# Alias execute function
def _send(msg, **kwargs):
"""Send a message to the kernel client."""
# Include dedent of msg so we don't get odd indentation errors.
return self.kc.execute(textwrap.dedent(msg), **kwargs)
self.send = _send
# Ping the kernel
self.kc.kernel_info()
try:
self.reply = self.kc.get_shell_msg(timeout=1)
except Empty:
continue
else:
connected = True
if connected:
# Send command so that monitor knows vim is commected
# send('"_vim_client"', store_history=False)
self.pid = self.set_pid() # Ask kernel for its PID
self.vim_echom('kernel connection successful! pid = {}'.format(
self.pid),
style='Question')
else:
self.kc.stop_channels()
self.vim_echom('kernel connection attempt timed out',
style='Error')
def _find_connection_file(connection_file):
"""Return the absolute path for a connection file
- If nothing specified, return current Kernel's connection file
- Otherwise, call jupyter_client.find_connection_file
"""
if connection_file is None:
# get connection file from current kernel
return get_connection_file()
else:
return jupyter_client.find_connection_file(connection_file)
def connect(self, info):
"""Connect to an existing kernel to inspect.
info should either be a dict with connection info, or a kernel
file name like "kernel-<GUID>.json". This information can normally
be gotten by the magic `%connect_info`.
"""
if isinstance(info, dict):
self.client.load_connection_info(info)
else:
path = jupyter_client.find_connection_file(info)
self.client.load_connection_file(path)
def send_to(self, args):
if args and args[0].endswith('(newest)'):
args[0] = args[0][:-len('(newest)')]
cf = find_connection_file(*args)
if cf not in self.clients:
client = BlockingKernelClient()
client.load_connection_file(cf)
client.start_channels()
self.clients[cf] = client
return cf
def execute_from_command_line():
if sys.argv.count('--existing') != 1:
raise ValueError(f'{sys.argv}\n'
f'--existing argument must occur once only.')
kernel_arg_index = sys.argv.index('--existing')
try:
kernel_name = sys.argv[kernel_arg_index + 1]
except IndexError:
# Following the command-line API of jupyter console, qtconsole etc, the --existing argument
# can be used without a value, meaning use the kernel whose connection file has most
# recently been accessed. We support that here when --existing is the last element of the
# command line. Otherwise, the behavior of the no-argument-value form can be achieved with
# --existing ''.
kernel_name = None
else:
sys.argv.pop(kernel_arg_index + 1)
sys.argv.pop(kernel_arg_index)
if {'shell', 'shell_plus'} & set(sys.argv):
# Special case: use `jupyter console` for management commands requesting a python shell.
argv = [
'jupyter', 'console', '--Completer.use_jedi=False', '--existing'
]
if kernel_name:
argv.append(kernel_name)
os.execlp(argv[0], *argv)
connection_file = find_connection_file(
kernel_name) if kernel_name else find_connection_file()
kernel_client = BlockingKernelClient(connection_file=connection_file)
kernel_client.load_connection_file()
response = kernel_client.execute_interactive(f"""
from devkernel.kernel import execute_from_command_line
execute_from_command_line('{json.dumps(sys.argv)}')
""")
exit_status = 0 if response['metadata']['status'] == 'ok' else 1
sys.exit(exit_status)
def connect_to_kernel():
"""Create kernel manager from existing connection file."""
from jupyter_client import KernelManager, find_connection_file
global kc, pid, send
# Test if connection is alive
connected = check_connection()
attempt = 0
max_attempts = 3
while not connected and attempt < max_attempts:
attempt += 1
try:
cfile = find_connection_file() # default filename='kernel-*.json'
except IOError:
vim_echom("kernel connection attempt {:d} failed - no kernel file"\
.format(attempt), style="Error")
continue
# Create the kernel manager and connect a client
# See: <http://jupyter-client.readthedocs.io/en/stable/api/client.html>
km = KernelManager(connection_file=cfile)
km.load_connection_file()
kc = km.client()
kc.start_channels()
# Alias execute function
def _send(msg, **kwargs):
"""Send a message to the kernel client."""
# Include dedent of msg so we don't get odd indentation errors.
return kc.execute(textwrap.dedent(msg), **kwargs)
send = _send
# Ping the kernel
kc.kernel_info()
try:
reply = kc.get_shell_msg(timeout=1)
except Empty:
continue
else:
connected = True
if connected:
# Send command so that monitor knows vim is commected
# send('"_vim_client"', store_history=False)
pid = set_pid() # Ask kernel for its PID
vim_echom('kernel connection successful! pid = {}'.format(pid),
style='Question')
else:
kc.stop_channels()
vim_echom('kernel connection attempt timed out', style='Error')
def register_volume(self, volume):
# globally register volume
global volumes
volumes[volume.token] = volume
# globally register kernel client for this volume in the Jupyter server
cf = url_escape(find_connection_file())
http_client= HTTPClient()
try:
response = http_client.fetch(self.get_server_url() + '/register_token/' + volume.token.decode('utf8') + '/' + cf)
except Exception as e:
raise RuntimeError("could not register token: " + str(e))
http_client.close()
def get_console_info():
from jupyter_client import find_connection_file
import socket
from micropsi_server.micropsi_app import console_is_started
if not console_is_started:
return None
connection_file = find_connection_file()
with open(connection_file) as connection_info_file:
connection_info = json.load(connection_info_file)
if connection_info["ip"] != "127.0.0.1":
external_ip = socket.gethostbyname(socket.gethostname())
connection_info["ip"] = external_ip
return connection_info
return None
def spawn_qt(self, bv):
argv = []
cf = jupyter_client.find_connection_file(self.connection_file)
cmd = ';'.join([
"from IPython.qt.console import qtconsoleapp",
"qtconsoleapp.main()"
])
kwargs = {}
kwargs['start_new_session'] = True
Popen(['python', '-c', cmd, '--existing', cf] + argv,
stdout=PIPE,
stderr=PIPE,
close_fds=(sys.platform != 'win32'),
**kwargs)
def find_connection_file(filename='kernel-*.json', profile=None):
"""find a connection file, and return its absolute path.
The current working directory and the profile's security
directory will be searched for the file if it is not given by
absolute path.
If profile is unspecified, then the current running application's
profile will be used, or 'default', if not run from IPython.
If the argument does not match an existing file, it will be interpreted as a
fileglob, and the matching file in the profile's security dir with
the latest access time will be used.
Parameters
----------
filename : str
The connection file or fileglob to search for.
profile : str [optional]
The name of the profile to use when searching for the connection file,
if different from the current IPython session or 'default'.
Returns
-------
str : The absolute path of the connection file.
"""
from IPython.core.application import BaseIPythonApplication as IPApp
try:
# quick check for absolute path, before going through logic
return filefind(filename)
except IOError:
pass
if profile is None:
# profile unspecified, check if running from an IPython app
if IPApp.initialized():
app = IPApp.instance()
profile_dir = app.profile_dir
else:
# not running in IPython, use default profile
profile_dir = ProfileDir.find_profile_dir_by_name(
get_ipython_dir(), 'default')
else:
# find profiledir by profile name:
profile_dir = ProfileDir.find_profile_dir_by_name(
get_ipython_dir(), profile)
security_dir = profile_dir.security_dir
return jupyter_client.find_connection_file(filename,
path=['.', security_dir])
def startup_connection(self, ipykernel_filename):
self.kernel_manager = jupyter_client.KernelManager(
connection_file=jupyter_client.find_connection_file(
ipykernel_filename,
path=[
".",
path.join("/", "run", "user", str(getuid()), "jupyter"),
path.join(path.expanduser("~"), ".ipython",
"profile_default", "security")
],
))
self.kernel_manager.load_connection_file()
self.client = self.kernel_manager.blocking_client()
self.client.start_channels()
reply = self.client.get_shell_msg()
def main(kid, var):
# Load connection info and init communications.
cf = find_connection_file(kid) # str(port))
km = BlockingKernelClient(connection_file=cf)
km.load_connection_file()
km.start_channels()
code = f"""
import pandas as pd
import numpy as np
if type({var}) in [pd.DataFrame, np.ndarray, list]:
print({var}.to_json(orient='split', index=False))
"""
km.execute_interactive(code, timeout=TIMEOUT)
km.stop_channels()
def find_connection_file(filename='kernel-*.json', profile=None):
"""DEPRECATED: find a connection file, and return its absolute path.
THIS FUNCTION IS DEPRECATED. Use juptyer_client.find_connection_file instead.
Parameters
----------
filename : str
The connection file or fileglob to search for.
profile : str [optional]
The name of the profile to use when searching for the connection file,
if different from the current IPython session or 'default'.
Returns
-------
str : The absolute path of the connection file.
"""
import warnings
warnings.warn(
"""ipykernel.find_connection_file is deprecated, use jupyter_client.find_connection_file""",
DeprecationWarning,
stacklevel=2)
from IPython.core.application import BaseIPythonApplication as IPApp
try:
# quick check for absolute path, before going through logic
return filefind(filename)
except IOError:
pass
if profile is None:
# profile unspecified, check if running from an IPython app
if IPApp.initialized():
app = IPApp.instance()
profile_dir = app.profile_dir
else:
# not running in IPython, use default profile
profile_dir = ProfileDir.find_profile_dir_by_name(
get_ipython_dir(), 'default')
else:
# find profiledir by profile name:
profile_dir = ProfileDir.find_profile_dir_by_name(
get_ipython_dir(), profile)
security_dir = profile_dir.security_dir
return jupyter_client.find_connection_file(filename,
path=['.', security_dir])
def find_connection_file(filename='kernel-*.json', profile=None):
"""find a connection file, and return its absolute path.
The current working directory and the profile's security
directory will be searched for the file if it is not given by
absolute path.
If profile is unspecified, then the current running application's
profile will be used, or 'default', if not run from IPython.
If the argument does not match an existing file, it will be interpreted as a
fileglob, and the matching file in the profile's security dir with
the latest access time will be used.
Parameters
----------
filename : str
The connection file or fileglob to search for.
profile : str [optional]
The name of the profile to use when searching for the connection file,
if different from the current IPython session or 'default'.
Returns
-------
str : The absolute path of the connection file.
"""
from IPython.core.application import BaseIPythonApplication as IPApp
try:
# quick check for absolute path, before going through logic
return filefind(filename)
except IOError:
pass
if profile is None:
# profile unspecified, check if running from an IPython app
if IPApp.initialized():
app = IPApp.instance()
profile_dir = app.profile_dir
else:
# not running in IPython, use default profile
profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')
else:
# find profiledir by profile name:
profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
security_dir = profile_dir.security_dir
return jupyter_client.find_connection_file(filename, path=['.', security_dir])
def kernel(f):
if verbose:
logging.info("kernel connecting " + f)
cf = jupyter_client.find_connection_file(f)
km = jupyter_client.BlockingKernelClient(connection_file=cf)
# load connection info and init communication
km.load_connection_file()
km.start_channels()
try:
km.wait_for_ready(timeout=network_timeout)
except:
logging.error("he's dead, jim")
sys.exit(1)
return km
def register_volume(self, volume):
# globally register volume
global volumes
volumes[volume.token] = volume
# globally register kernel client for this volume in the Jupyter server
cf = url_escape(find_connection_file())
http_client = HTTPClient()
try:
response = http_client.fetch(self.get_server_url() +
'/register_token/' +
volume.token.decode('utf8') + '/' +
cf)
except Exception as e:
raise RuntimeError("could not register token: " + str(e))
http_client.close()
def thread_connect_to_kernel(self):
"""Create kernel manager from existing connection file (Async)."""
if self.sync.check_stop():
return
# Check if connection is alive
connected = self.kernel_client.check_connection()
# Try to connect
max_attempts = 3
for attempt in range(max_attempts):
# NOTE if user tries to :JupyterConnect <new_pid>, this check will ignore
# the requested new pid.
if connected:
break
# Check if thread want to return
if self.sync.check_stop():
return
# Find connection file
try:
self.kernel_client.cfile = find_connection_file(
filename=self.kernel_client.kernel_info['cfile_user'])
except IOError:
self.vim_client.thread_echom(
"kernel connection attempt {:d}/{:d} failed - no kernel file"
.format(attempt, max_attempts),
style="Error")
continue
# Connect
connected = self.kernel_client.create_kernel_manager()
# Early return if failed
if not connected:
self.kernel_client.disconnnect()
self.vim_client.thread_echom('kernel connection attempt timed out',
style='Error')
return
# Collect and echom kernel info
self.vim_client.thread_echom_kernel_info(
self.kernel_client.get_kernel_info(self.lang))
# Inform everything ok, at last
self.vim_client.thread_echom('Connected! ', style='Question')
def startup_connection(self, ipykernel_filename):
self.kernel_manager = jupyter_client.KernelManager(
connection_file=jupyter_client.find_connection_file(
ipykernel_filename,
path=[
".",
path.join("/", "run", "user", str(getuid()), "jupyter"),
path.join(
path.expanduser("~"),
".ipython", "profile_default", "security"
)
],
)
)
self.kernel_manager.load_connection_file()
self.client = self.kernel_manager.blocking_client()
self.client.start_channels()
reply = self.client.get_shell_msg()
def find_connection_file(filename='kernel-*.json', profile=None):
"""DEPRECATED: find a connection file, and return its absolute path.
THIS FUNCION IS DEPRECATED. Use juptyer_client.find_connection_file instead.
Parameters
----------
filename : str
The connection file or fileglob to search for.
profile : str [optional]
The name of the profile to use when searching for the connection file,
if different from the current IPython session or 'default'.
Returns
-------
str : The absolute path of the connection file.
"""
import warnings
warnings.warn("""ipykernel.find_connection_file is deprecated, use jupyter_client.find_connection_file""",
DeprecationWarning, stacklevel=2)
from IPython.core.application import BaseIPythonApplication as IPApp
try:
# quick check for absolute path, before going through logic
return filefind(filename)
except IOError:
pass
if profile is None:
# profile unspecified, check if running from an IPython app
if IPApp.initialized():
app = IPApp.instance()
profile_dir = app.profile_dir
else:
# not running in IPython, use default profile
profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')
else:
# find profiledir by profile name:
profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
security_dir = profile_dir.security_dir
return jupyter_client.find_connection_file(filename, path=['.', security_dir])
def open(self, url_component=None):
"""Websocket connection opened.
Call our terminal manager to get a terminal, and connect to it as a
client.
"""
self._log.info("KernelSocket.open: %s", url_component)
with open(find_connection_file()) as fid:
self.info = json.load(fid)
self._log.info(self.info)
ctx = zmq.Context()
self.sock = ctx.socket(zmq.SUB)
self.sock.connect('tcp://%s:%s' % (self.info['ip'],
self.info['iopub_port']))
self.sock.setsockopt(zmq.SUBSCRIBE, b'')
self.stream = ZMQStream(self.sock)
self.stream.on_recv(self.on_recv)
self._log.info("KernelSocket.open: Opened %s", self.info['ip'])
def _find_connection_file(connection_file, profile=None):
"""Return the absolute path for a connection file
- If nothing specified, return current Kernel's connection file
- If profile specified, show deprecation warning about finding connection files in profiles
- Otherwise, call jupyter_client.find_connection_file
"""
if connection_file is None:
# get connection file from current kernel
return get_connection_file()
else:
# connection file specified, allow shortnames:
if profile is not None:
warnings.warn(
"Finding connection file by profile is deprecated.",
DeprecationWarning, stacklevel=3,
)
return find_connection_file(connection_file, profile=profile)
else:
return jupyter_client.find_connection_file(connection_file)
def try_find(s):
try:
return os.path.isfile(find_connection_file(s))
except Exception:
return False
def initialize(self):
self._log = logging.getLogger(__name__)
#self._log.setLevel(logging.INFO)
with open(find_connection_file()) as fid:
self.info = json.load(fid)
self._log.info(self.info)
colors = {k: i for i, k in enumerate([
'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white'])}
def paths():
for fullpath in glob(os.path.join(os.path.dirname(filename), 'kernel*')):
if not re.match('^(.*/)?kernel-[0-9]+.json', fullpath):
continue
yield fullpath
connected = False
while not connected:
try:
filename = find_connection_file('kernel*')
except IOError:
continue
for fullpath in paths():
km = KernelManager(connection_file=fullpath)
km.load_connection_file()
kc = km.client()
kc.start_channels()
try:
send = kc.execute
except AttributeError:
send = kc.shell_channel.execute
if not hasattr(kc, 'iopub_channel'):
kc.iopub_channel = kc.sub_channel
def km_from_string(s=''):
"""create kernel manager from IPKernelApp string
such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
or just 'kernel-12345.json' for IPython 0.12
"""
try:
import jupyter_client
except ImportError:
raise ImportError("Could not find jupyter_client. " + _install_instructions)
from traitlets.config.loader import KeyValueConfigLoader
try:
from jupyter_client import find_connection_file
from jupyter_client.manager import KernelManager
except ImportError:
# IPython < 1.0
from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
from IPython.zmq.kernelapp import kernel_aliases
try:
from IPython.lib.kernel import find_connection_file
except ImportError:
# < 0.12, no find_connection_file
pass
global km, kc, send
s = s.replace('--existing', '')
if 'connection_file' in KernelManager.class_trait_names():
# 0.12 uses files instead of a collection of ports
# include default IPython search path
# filefind also allows for absolute paths, in which case the search
# is ignored
try:
# XXX: the following approach will be brittle, depending on what
# connection strings will end up looking like in the future, and
# whether or not they are allowed to have spaces. I'll have to sync
# up with the IPython team to address these issues -pi
if '--profile' in s:
k,p = s.split('--profile')
k = k.lstrip().rstrip() # kernel part of the string
p = p.lstrip().rstrip() # profile part of the string
fullpath = find_connection_file(k,p)
else:
fullpath = find_connection_file(s.lstrip().rstrip())
except IOError as e:
echo(":IPython " + s + " failed", "Info")
echo("^-- failed '" + s + "' not found", "Error")
return
km = KernelManager(connection_file = fullpath)
km.load_connection_file()
else:
if s == '':
echo(":IPython 0.11 requires the full connection string")
return
loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
cfg = loader.load_config()['KernelApp']
try:
km = KernelManager(
shell_address=(ip, cfg['shell_port']),
sub_address=(ip, cfg['iopub_port']),
stdin_address=(ip, cfg['stdin_port']),
hb_address=(ip, cfg['hb_port']))
except KeyError as e:
echo(":IPython " +s + " failed", "Info")
echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
return
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
try:
send = kc.execute
except AttributeError:
# < 3.0
send = kc.shell_channel.execute
# now that we're connect to an ipython kernel, activate completion
# machinery, but do so only for the local buffer if the user added the
# following line the vimrc:
# let g:ipy_completefunc = 'local'
vim.command("""
if g:ipy_completefunc == 'global'
set completefunc=CompleteIPython
elseif g:ipy_completefunc == 'local'
setl completefunc=CompleteIPython
endif
""")
# also activate GUI doc balloons if in gvim
vim.command("""
if has('balloon_eval')
set bexpr=IPythonBalloonExpr()
endif
""")
set_pid()
return km
def km_from_string(s=''):
"""create kernel manager from IPKernelApp string
such as '--shell=47378 --iopub=39859 --stdin=36778 --hb=52668' for IPython 0.11
or just 'kernel-12345.json' for IPython 0.12
"""
try:
import IPython
except ImportError:
raise ImportError("Could not find IPython. " + _install_instructions)
try:
from traitlets.config.loader import KeyValueConfigLoader
except ImportError:
from IPython.config.loader import KeyValueConfigLoader
try:
from jupyter_client import KernelManager, find_connection_file
except ImportError:
try:
from IPython.kernel import KernelManager, find_connection_file
except ImportError:
# IPython < 1.0
from IPython.zmq.blockingkernelmanager import BlockingKernelManager as KernelManager
from IPython.zmq.kernelapp import kernel_aliases
try:
from IPython.lib.kernel import find_connection_file
except ImportError:
# < 0.12, no find_connection_file
pass
global km, kc, send, history, complete, object_info
# Test if connection is still alive
connected = False
starttime = time.time()
attempt = 0
s = s.replace('--existing', '')
while not connected and (time.time() - starttime) < 5.0:
if not attempt and os.path.isfile(s):
fullpath = s
else:
try:
s = fullpath = find_connection_file('kernel*')
except IOError:
echo("IPython connection attempt #%d failed - no kernel file" % attempt, "Warning")
time.sleep(1)
continue
attempt += 1
km = KernelManager(connection_file=fullpath)
km.load_connection_file()
kc = km.client()
kc.start_channels()
if 'connection_file' in KernelManager.class_trait_names():
# 0.12 uses files instead of a collection of ports
# include default IPython search path
# filefind also allows for absolute paths, in which case the search
# is ignored
try:
# XXX: the following approach will be brittle, depending on what
# connection strings will end up looking like in the future, and
# whether or not they are allowed to have spaces. I'll have to sync
# up with the IPython team to address these issues -pi
if '--profile' in s:
k,p = s.split('--profile')
k = k.lstrip().rstrip() # kernel part of the string
p = p.lstrip().rstrip() # profile part of the string
fullpath = find_connection_file(k,p)
else:
fullpath = find_connection_file(s.lstrip().rstrip())
except IOError as e:
echo(":IPython " + s + " failed", "Info")
echo("^-- failed '" + s + "' not found", "Error")
return
km = KernelManager(connection_file = fullpath)
km.load_connection_file()
else:
if s == '':
echo(":IPython 0.11 requires the full connection string")
return
loader = KeyValueConfigLoader(s.split(), aliases=kernel_aliases)
cfg = loader.load_config()['KernelApp']
try:
km = KernelManager(
shell_address=(ip, cfg['shell_port']),
sub_address=(ip, cfg['iopub_port']),
stdin_address=(ip, cfg['stdin_port']),
hb_address=(ip, cfg['hb_port']))
except KeyError as e:
echo(":IPython " +s + " failed", "Info")
echo("^-- failed --"+e.message.replace('_port','')+" not specified", "Error")
return
try:
kc = km.client()
except AttributeError:
# 0.13
kc = km
kc.start_channels()
execute = kc.execute if hasattr(kc, 'execute') else kc.shell_channel.execute
history = kc.history if hasattr(kc, 'history') else kc.shell_channel.history
complete = kc.complete if hasattr(kc, 'complete') else kc.shell_channel.complete
object_info = kc.inspect if hasattr(kc, 'inspect') else kc.shell_channel.object_info
def send(msg, **kwargs):
kwds = dict(
store_history=vim_vars.get('ipython_store_history', True),
)
kwds.update(kwargs)
return execute(msg, **kwds)
send('', silent=True)
try:
msg = kc.shell_channel.get_msg(timeout=1)
connected = True
except:
echo("IPython connection attempt #%d failed - no messages" % attempt, "Warning")
continue
#XXX: backwards compatibility for IPython < 1.0
if not hasattr(kc, 'iopub_channel'):
kc.iopub_channel = kc.sub_channel
set_pid()
if not connected:
echo("IPython connection attempt timed out", "Error")
return
else:
vim.command('redraw')
echo("IPython connection successful")
send('"_vim_client";_=_;__=__', store_history=False)
#XXX: backwards compatibility for IPython < 0.13
sc = kc.shell_channel
if hasattr(sc, 'object_info'):
import inspect
num_oinfo_args = len(inspect.getargspec(sc.object_info).args)
if num_oinfo_args == 2:
# patch the object_info method which used to only take one argument
klass = sc.__class__
klass._oinfo_orig = klass.object_info
klass.object_info = lambda s,x,y: s._oinfo_orig(x)
#XXX: backwards compatibility for IPython < 1.0
if not hasattr(kc, 'iopub_channel'):
kc.iopub_channel = kc.sub_channel
# now that we're connect to an ipython kernel, activate completion
# machinery, but do so only for the local buffer if the user added the
# following line the vimrc:
# let g:ipy_completefunc = 'local'
vim.command("""
if g:ipy_completefunc == 'global'
set completefunc=CompleteIPython
elseif g:ipy_completefunc == 'local'
setl completefunc=CompleteIPython
elseif g:ipy_completefunc == 'omni'
setl omnifunc=CompleteIPython
endif
""")
# also activate GUI doc balloons if in gvim
vim.command("""
if has('balloon_eval')
set bexpr=IPythonBalloonExpr()
endif
""")
return km
print('msg = %s' % str(msg))
# Alias some functions to attributes (IPython names changed)
execute_input = pyin
execute_result = pyout
error = pyerr
#------------------------------------------------------------------------------
# Connect to the kernel
#------------------------------------------------------------------------------
# TODO move this loop to __init__ of IPythonMonitor??
connected = False
while not connected:
try:
# Default: filename='kernel-*.json'
filename = find_connection_file()
except IOError:
continue
# Create the kernel manager and connect a client
km = KernelManager(connection_file=filename)
km.load_connection_file()
kc = km.client()
kc.start_channels()
# Ping the kernel
msg_id = kc.kernel_info()
try:
reply = kc.get_shell_msg(timeout=1)
except Empty:
continue
