class Kernel(object): # kernel config is stored in a dot file with the active directory def __init__(self, config, active_dir, pyspark): # right now we're spawning a child process for IPython. we can # probably work directly with the IPython kernel API, but the docs # don't really explain how to do it. log_file = None if pyspark: os.environ["IPYTHON_OPTS"] = "kernel -f %s" % config pyspark = os.path.join(os.environ.get("SPARK_HOME"), "bin/pyspark") spark_log = os.environ.get("SPARK_LOG", None) if spark_log: log_file = open(spark_log, "w") spark_opts = os.environ.get("SPARK_OPTS", "") args = [pyspark] + spark_opts.split() # $SPARK_HOME/bin/pyspark <SPARK_OPTS> p = subprocess.Popen(args, stdout=log_file, stderr=log_file) else: args = [sys.executable, '-m', 'IPython', 'kernel', '-f', config] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # when __this__ process exits, we're going to remove the ipython config # file and kill the ipython subprocess atexit.register(p.terminate) def remove_config(): if os.path.isfile(config): os.remove(config) atexit.register(remove_config) # i found that if i tried to connect to the kernel immediately, so we'll # wait until the config file exists before moving on while os.path.isfile(config)==False: time.sleep(0.1) def close_file(): if log_file: log_file.close() atexit.register(close_file) # fire up the kernel with the appropriate config self.client = BlockingKernelClient(connection_file=config) self.client.load_connection_file() self.client.start_channels() # load our monkeypatches... self.client.execute("%matplotlib inline") python_patch_file = os.path.join(dirname, "langs", "python-patch.py") self.client.execute("%run " + python_patch_file) def _run_code(self, execution_id, code, timeout=0.1): # this function executes some code and waits for it to completely finish # before returning. i don't think that this is neccessarily the best # way to do this, but the IPython documentation isn't very helpful for # this particular topic. # # 1) execute code and grab the ID for that execution thread # 2) look for messages coming from the "iopub" channel (this is just a # stream of output) # 3) when we get a message that is one of the following, save relevant # data to `data`: # - execute_result - content from repr # - stream - content from stdout # - error - ansii encoded stacktrace # the final piece is that we check for when the message indicates that # the kernel is idle and the message's parent is the original execution # ID (msg_id) that's associated with our executing code. if this is the # case, we'll return the data and the msg_id and exit msg_id = self.client.execute(code, allow_stdin=False) request = { "id": execution_id, "msg_id": msg_id, "code": code, "status": "started" } sys.stdout.write(json.dumps(request) + '\n') sys.stdout.flush() output = { "id": execution_id, "msg_id": msg_id, "output": "", "stream": None, "image": None, "error": None } while True: try: reply = self.client.get_iopub_msg(timeout=timeout) except Empty: continue if "execution_state" in reply['content']: if reply['content']['execution_state']=="idle" and reply['parent_header']['msg_id']==msg_id: if reply['parent_header']['msg_type']=="execute_request": request["status"] = "complete" sys.stdout.write(json.dumps(request) + '\n') sys.stdout.flush() return elif reply['header']['msg_type']=="execute_result": output['output'] = reply['content']['data'].get('text/plain', '') output['stream'] = reply['content']['data'].get('text/plain', '') elif reply['header']['msg_type']=="display_data": if 'image/png' in reply['content']['data']: output['image'] = reply['content']['data']['image/png'] elif 'text/html' in reply['content']['data']: output['html'] = reply['content']['data']['text/html'] elif reply['header']['msg_type']=="stream": output['output'] += reply['content'].get('text', '') output['stream'] = reply['content'].get('text', '') elif reply['header']['msg_type']=="error": output['error'] = "\n".join(reply['content']['traceback']) # TODO: if we have something non-trivial to send back... sys.stdout.write(json.dumps(output) + '\n') sys.stdout.flush() # TODO: should probably get rid of all this output['stream'] = None output['image'] = None output['html'] = None def _complete(self, execution_id, code, timeout=0.5): # Call ipython kernel complete, wait for response with the correct msg_id, # and construct appropriate UI payload. # See below for an example response from ipython kernel completion for 'el' # # { # 'parent_header': # {u'username': u'ubuntu', u'version': u'5.0', u'msg_type': u'complete_request', # u'msg_id': u'5222d158-ada8-474e-88d8-8907eb7cc74c', u'session': u'cda4a03d-a8a1-4e6c-acd0-de62d169772e', # u'date': datetime.datetime(2015, 5, 7, 15, 25, 8, 796886)}, # 'msg_type': u'complete_reply', # 'msg_id': u'a3a957d6-5865-4c6f-a0b2-9aa8da718b0d', # 'content': # {u'matches': [u'elif', u'else'], u'status': u'ok', u'cursor_start': 0, u'cursor_end': 2, u'metadata': {}}, # 'header': # {u'username': u'ubuntu', u'version': u'5.0', u'msg_type': u'complete_reply', # u'msg_id': u'a3a957d6-5865-4c6f-a0b2-9aa8da718b0d', u'session': u'f1491112-7234-4782-8601-b4fb2697a2f6', # u'date': datetime.datetime(2015, 5, 7, 15, 25, 8, 803470)}, # 'buffers': [], # 'metadata': {} # } # msg_id = self.client.complete(code) request = { "id": execution_id, "msg_id": msg_id, "code": code, "status": "started" } sys.stdout.write(json.dumps(request) + '\n') sys.stdout.flush() output = { "id": execution_id, "msg_id": msg_id, "output": None, "image": None, "error": None } while True: try: reply = self.client.get_shell_msg(timeout=timeout) except Empty: continue if "matches" in reply['content'] and reply['msg_type']=="complete_reply" and reply['parent_header']['msg_id']==msg_id: results = [] for completion in reply['content']['matches']: result = { "value": completion, "dtype": "---" } if "." in code: # result['text'] = result['value'] # ".".join(result['value'].split(".")[1:]) result['text'] = result['value'] #.split('.')[-1] result["dtype"] = "function" else: result['text'] = result['value'] result["dtype"] = "" # type(globals().get(code)).__name__ results.append(result) output['output'] = results output['status'] = "complete" sys.stdout.write(json.dumps(output) + '\n') sys.stdout.flush() return def execute(self, execution_id, code, complete=False): if complete==True: return self._complete(execution_id, code) else: result = self._run_code(execution_id, code) if re.match("%?reset", code): # load our monkeypatches... k.client.execute("%matplotlib inline") k.client.execute(vars_patch) return result def get_packages(self): return self.execute("__get_packages()")
class SendToIPython(object): def __init__(self, nvim): self.nvim = nvim self.client = None self.kerneldir = Path(jupyter_runtime_dir()) @neovim.function('RunningKernels', sync=True) def running_kernels(self, args): l = self.kerneldir.glob('kernel-*.json') l = sorted(l, reverse=True, key=lambda f: f.stat().st_ctime) return [f.name for f in l] @neovim.command('SendTo', complete='customlist,RunningKernels', nargs='?') def send_to(self, args): cfs = args or self.running_kernels(None) if not cfs: self.nvim.command('echom "No kernel found"') return if self.client is not None: self.client.stop_channels() cf = cfs[0] self.client = BlockingKernelClient() self.client.load_connection_file(self.kerneldir / cf) self.client.start_channels() # run function once to register it for the `funcref` function self.nvim.command('call SendLinesToJupyter()') self.nvim.command( 'let g:send_target = {"send": funcref("SendLinesToJupyter")}') self.nvim.command('echom "Sending to %s"' % cf) @neovim.function('SendLinesToJupyter') def send_lines(self, args): if args: self.client.execute('\n'.join(args[0])) @neovim.function('SendComplete', sync=True) def complete(self, args): findstart, base = args if self.client is None: return -3 # no client setup yet: cancel silently and leave completion mode if findstart: line = self.nvim.current.line if not line: return -2 # empty line: cancel silently but stay in completion mode pos = self.nvim.current.window.cursor[1] try: reply = self.client.complete(line, pos, reply=True, timeout=timeout)['content'] except TimeoutError: return -2 self.completions = [{ 'word': w, 'info': ' ' } for w in reply['matches']] return reply['cursor_start'] else: # TODO: use vim's complete_add/complete_check for async operation get_info(self.client, self.completions) return {'words': self.completions, 'refresh': 'always'} @neovim.function('SendCanComplete', sync=True) def can_complete(self, args): return args[ 0] != '' and self.client is not None and self.client.is_alive()