class Kernel(object):
# kernel config is stored in a dot file with the active directory
def __init__(self, config, active_dir, pyspark):
# right now we're spawning a child process for IPython. we can
# probably work directly with the IPython kernel API, but the docs
# don't really explain how to do it.
log_file = None
if pyspark:
os.environ["IPYTHON_OPTS"] = "kernel -f %s" % config
pyspark = os.path.join(os.environ.get("SPARK_HOME"), "bin/pyspark")
spark_log = os.environ.get("SPARK_LOG", None)
if spark_log:
log_file = open(spark_log, "w")
spark_opts = os.environ.get("SPARK_OPTS", "")
args = [pyspark] + spark_opts.split() # $SPARK_HOME/bin/pyspark <SPARK_OPTS>
p = subprocess.Popen(args, stdout=log_file, stderr=log_file)
else:
args = [sys.executable, '-m', 'IPython', 'kernel', '-f', config]
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# when __this__ process exits, we're going to remove the ipython config
# file and kill the ipython subprocess
atexit.register(p.terminate)
def remove_config():
if os.path.isfile(config):
os.remove(config)
atexit.register(remove_config)
# i found that if i tried to connect to the kernel immediately, so we'll
# wait until the config file exists before moving on
while os.path.isfile(config)==False:
time.sleep(0.1)
def close_file():
if log_file:
log_file.close()
atexit.register(close_file)
# fire up the kernel with the appropriate config
self.client = BlockingKernelClient(connection_file=config)
self.client.load_connection_file()
self.client.start_channels()
# load our monkeypatches...
self.client.execute("%matplotlib inline")
python_patch_file = os.path.join(dirname, "langs", "python-patch.py")
self.client.execute("%run " + python_patch_file)
def _run_code(self, execution_id, code, timeout=0.1):
# this function executes some code and waits for it to completely finish
# before returning. i don't think that this is neccessarily the best
# way to do this, but the IPython documentation isn't very helpful for
# this particular topic.
#
# 1) execute code and grab the ID for that execution thread
# 2) look for messages coming from the "iopub" channel (this is just a
# stream of output)
# 3) when we get a message that is one of the following, save relevant
# data to `data`:
# - execute_result - content from repr
# - stream - content from stdout
# - error - ansii encoded stacktrace
# the final piece is that we check for when the message indicates that
# the kernel is idle and the message's parent is the original execution
# ID (msg_id) that's associated with our executing code. if this is the
# case, we'll return the data and the msg_id and exit
msg_id = self.client.execute(code, allow_stdin=False)
request = { "id": execution_id, "msg_id": msg_id, "code": code, "status": "started" }
sys.stdout.write(json.dumps(request) + '\n')
sys.stdout.flush()
output = {
"id": execution_id,
"msg_id": msg_id,
"output": "",
"stream": None,
"image": None,
"error": None
}
while True:
try:
reply = self.client.get_iopub_msg(timeout=timeout)
except Empty:
continue
if "execution_state" in reply['content']:
if reply['content']['execution_state']=="idle" and reply['parent_header']['msg_id']==msg_id:
if reply['parent_header']['msg_type']=="execute_request":
request["status"] = "complete"
sys.stdout.write(json.dumps(request) + '\n')
sys.stdout.flush()
return
elif reply['header']['msg_type']=="execute_result":
output['output'] = reply['content']['data'].get('text/plain', '')
output['stream'] = reply['content']['data'].get('text/plain', '')
elif reply['header']['msg_type']=="display_data":
if 'image/png' in reply['content']['data']:
output['image'] = reply['content']['data']['image/png']
elif 'text/html' in reply['content']['data']:
output['html'] = reply['content']['data']['text/html']
elif reply['header']['msg_type']=="stream":
output['output'] += reply['content'].get('text', '')
output['stream'] = reply['content'].get('text', '')
elif reply['header']['msg_type']=="error":
output['error'] = "\n".join(reply['content']['traceback'])
# TODO: if we have something non-trivial to send back...
sys.stdout.write(json.dumps(output) + '\n')
sys.stdout.flush()
# TODO: should probably get rid of all this
output['stream'] = None
output['image'] = None
output['html'] = None
def _complete(self, execution_id, code, timeout=0.5):
# Call ipython kernel complete, wait for response with the correct msg_id,
# and construct appropriate UI payload.
# See below for an example response from ipython kernel completion for 'el'
#
# {
# 'parent_header':
# {u'username': u'ubuntu', u'version': u'5.0', u'msg_type': u'complete_request',
# u'msg_id': u'5222d158-ada8-474e-88d8-8907eb7cc74c', u'session': u'cda4a03d-a8a1-4e6c-acd0-de62d169772e',
# u'date': datetime.datetime(2015, 5, 7, 15, 25, 8, 796886)},
# 'msg_type': u'complete_reply',
# 'msg_id': u'a3a957d6-5865-4c6f-a0b2-9aa8da718b0d',
# 'content':
# {u'matches': [u'elif', u'else'], u'status': u'ok', u'cursor_start': 0, u'cursor_end': 2, u'metadata': {}},
# 'header':
# {u'username': u'ubuntu', u'version': u'5.0', u'msg_type': u'complete_reply',
# u'msg_id': u'a3a957d6-5865-4c6f-a0b2-9aa8da718b0d', u'session': u'f1491112-7234-4782-8601-b4fb2697a2f6',
# u'date': datetime.datetime(2015, 5, 7, 15, 25, 8, 803470)},
# 'buffers': [],
# 'metadata': {}
# }
#
msg_id = self.client.complete(code)
request = { "id": execution_id, "msg_id": msg_id, "code": code, "status": "started" }
sys.stdout.write(json.dumps(request) + '\n')
sys.stdout.flush()
output = { "id": execution_id, "msg_id": msg_id, "output": None, "image": None, "error": None }
while True:
try:
reply = self.client.get_shell_msg(timeout=timeout)
except Empty:
continue
if "matches" in reply['content'] and reply['msg_type']=="complete_reply" and reply['parent_header']['msg_id']==msg_id:
results = []
for completion in reply['content']['matches']:
result = {
"value": completion,
"dtype": "---"
}
if "." in code:
# result['text'] = result['value'] # ".".join(result['value'].split(".")[1:])
result['text'] = result['value'] #.split('.')[-1]
result["dtype"] = "function"
else:
result['text'] = result['value']
result["dtype"] = "" # type(globals().get(code)).__name__
results.append(result)
output['output'] = results
output['status'] = "complete"
sys.stdout.write(json.dumps(output) + '\n')
sys.stdout.flush()
return
def execute(self, execution_id, code, complete=False):
if complete==True:
return self._complete(execution_id, code)
else:
result = self._run_code(execution_id, code)
if re.match("%?reset", code):
# load our monkeypatches...
k.client.execute("%matplotlib inline")
k.client.execute(vars_patch)
return result
def get_packages(self):
return self.execute("__get_packages()")
class SendToIPython(object):
def __init__(self, nvim):
self.nvim = nvim
self.client = None
self.kerneldir = Path(jupyter_runtime_dir())
@neovim.function('RunningKernels', sync=True)
def running_kernels(self, args):
l = self.kerneldir.glob('kernel-*.json')
l = sorted(l, reverse=True, key=lambda f: f.stat().st_ctime)
return [f.name for f in l]
@neovim.command('SendTo', complete='customlist,RunningKernels', nargs='?')
def send_to(self, args):
cfs = args or self.running_kernels(None)
if not cfs:
self.nvim.command('echom "No kernel found"')
return
if self.client is not None:
self.client.stop_channels()
cf = cfs[0]
self.client = BlockingKernelClient()
self.client.load_connection_file(self.kerneldir / cf)
self.client.start_channels()
# run function once to register it for the `funcref` function
self.nvim.command('call SendLinesToJupyter()')
self.nvim.command(
'let g:send_target = {"send": funcref("SendLinesToJupyter")}')
self.nvim.command('echom "Sending to %s"' % cf)
@neovim.function('SendLinesToJupyter')
def send_lines(self, args):
if args:
self.client.execute('\n'.join(args[0]))
@neovim.function('SendComplete', sync=True)
def complete(self, args):
findstart, base = args
if self.client is None:
return -3 # no client setup yet: cancel silently and leave completion mode
if findstart:
line = self.nvim.current.line
if not line:
return -2 # empty line: cancel silently but stay in completion mode
pos = self.nvim.current.window.cursor[1]
try:
reply = self.client.complete(line,
pos,
reply=True,
timeout=timeout)['content']
except TimeoutError:
return -2
self.completions = [{
'word': w,
'info': ' '
} for w in reply['matches']]
return reply['cursor_start']
else:
# TODO: use vim's complete_add/complete_check for async operation
get_info(self.client, self.completions)
return {'words': self.completions, 'refresh': 'always'}
@neovim.function('SendCanComplete', sync=True)
def can_complete(self, args):
return args[
0] != '' and self.client is not None and self.client.is_alive()