def main(kid): # Load connection info and init communications. cf = find_connection_file(kid) km = BlockingKernelClient(connection_file=cf) km.load_connection_file() km.start_channels() # Define a function that is useful from within the user's notebook: juneau_connect() can be # used to directly connect the notebook to the source database. Note that this includes the # full "root" credentials. # FIXME: allow for user-specific credentials on SQL tables. The DBMS may also not be at localhost. code = f""" from sqlalchemy import create_engine def juneau_connect(): engine = create_engine( "postgresql://{config.sql.name}:{config.sql.password}@{config.sql.host}/{config.sql.dbname}", connect_args={{ "options": "-csearch_path='{config.sql.dbs}'" }} ) return engine.connect() """ km.execute_interactive(code, timeout=TIMEOUT) km.stop_channels()
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ kernel = Popen([sys.executable, "-c", cmd], stdout=PIPE, stderr=PIPE, env=env) connection_file = os.path.join(IPYTHONDIR, "profile_default", "security", "kernel-%i.json" % kernel.pid) # wait for connection file to exist, timeout after 5s tic = time.time() while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + SETUP_TIMEOUT: time.sleep(0.1) if kernel.poll() is not None: o, e = kernel.communicate() e = py3compat.cast_unicode(e) raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() client.wait_for_ready() try: yield client finally: client.stop_channels() kernel.terminate()
def exec_code(kid, code): """ Executes arbitrary `code` in the kernel with id `kid`. Returns: - tuple: the output of the code and the error, if any. """ # Load connection info and init communications. cf = find_connection_file(kid) with jupyter_lock: km = BlockingKernelClient(connection_file=cf) km.load_connection_file() km.start_channels() msg_id = km.execute(code, store_history=False) reply = km.get_shell_msg(msg_id, timeout=60) output, error = None, None while km.is_alive(): msg = km.get_iopub_msg(timeout=10) if ("content" in msg and "name" in msg["content"] and msg["content"]["name"] == "stdout"): output = msg["content"]["text"] break km.stop_channels() if reply["content"]["status"] != "ok": logging.error(f"Status is {reply['content']['status']}") logging.error(output) error = output output = None return output, error
def exec_code(kid, var, code): # load connection info and init communication cf = find_connection_file(kid) # str(port)) global jupyter_lock jupyter_lock.acquire() try: km = BlockingKernelClient(connection_file=cf) km.load_connection_file() km.start_channels() # logging.debug('Executing:\n' + str(code)) msg_id = km.execute(code, store_history=False) reply = km.get_shell_msg(msg_id, timeout=10) # logging.info('Execution reply:\n' + str(reply)) state = 'busy' output = None idle_count = 0 try: while km.is_alive(): try: msg = km.get_iopub_msg(timeout=10) # logging.debug('Read ' + str(msg)) if not 'content' in msg: continue if 'name' in msg['content'] and msg['content'][ 'name'] == 'stdout': # logging.debug('Got data '+ msg['content']['text']) output = msg['content']['text'] break if 'execution_state' in msg['content']: # logging.debug('Got state') state = msg['content']['execution_state'] if state == 'idle': idle_count = idle_count + 1 except Empty: pass except KeyboardInterrupt: logging.error('Keyboard interrupt') pass finally: # logging.info('Kernel IO finished') km.stop_channels() # logging.info(str(output)) error = '' if reply['content']['status'] != 'ok': logging.error('Status is ' + reply['content']['status']) logging.error(str(output)) error = output output = None finally: jupyter_lock.release() return output, error
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ def connection_file_ready(connection_file): """Check if connection_file is a readable json file.""" if not os.path.exists(connection_file): return False try: with open(connection_file) as f: json.load(f) return True except ValueError: return False kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE) try: connection_file = os.path.join( paths.jupyter_runtime_dir(), 'kernel-%i.json' % kernel.pid, ) # wait for connection file to exist, timeout after 5s tic = time.time() while not connection_file_ready(connection_file) \ and kernel.poll() is None \ and time.time() < tic + SETUP_TIMEOUT: time.sleep(0.1) # Wait 100ms for the writing to finish time.sleep(0.1) if kernel.poll() is not None: o,e = kernel.communicate() e = py3compat.cast_unicode(e) raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() client.wait_for_ready() try: yield client finally: client.stop_channels() finally: kernel.terminate()
def send_to(self, args): if args and args[0].endswith('(newest)'): args[0] = args[0][:-len('(newest)')] cf = find_connection_file(*args) if cf not in self.clients: client = BlockingKernelClient() client.load_connection_file(cf) client.start_channels() self.clients[cf] = client return cf
def run(self): cf = find_connection_file() client = BlockingKernelClient(connection_file=cf) client.load_connection_file() client.start_channels(shell=False, iopub=True, stdin=False, control=False, hb=False) while True: msg = client.get_iopub_msg() self.received.emit(msg)
def is_runing(cf): """ Check if kernel is alive. """ kc = BlockingKernelClient() kc.load_connection_file(cf) port = kc.get_connection_info()['iopub_port'] # if check_server(port): if is_open("127.0.0.1", port): return True else: return False
def connect(connection_file): if connection_file not in clients: print "[nyroglancer] connecting to: " + connection_file kernel_client = BlockingKernelClient(connection_file=connection_file) kernel_client.load_connection_file() kernel_client.start_channels() clients[connection_file] = kernel_client return kernel_client return clients[connection_file]
def connect(connection_file): if connection_file not in clients: print "[nyroglancer] connecting to: " + connection_file kernel_client = BlockingKernelClient(connection_file=connection_file) kernel_client.load_connection_file() kernel_client.start_channels() clients[connection_file] = kernel_client return kernel_client return clients[connection_file]
def main(kid, var): # Load connection info and init communications. cf = find_connection_file(kid) # str(port)) km = BlockingKernelClient(connection_file=cf) km.load_connection_file() km.start_channels() code = f""" import pandas as pd import numpy as np if type({var}) in [pd.DataFrame, np.ndarray, list]: print({var}.to_json(orient='split', index=False)) """ km.execute_interactive(code, timeout=TIMEOUT) km.stop_channels()
class PyExecutor(pyexecutor_pb2_grpc.PyExecutorServicer): def __init__(self): self.manager = MultiKernelManager() kernel_id = self.manager.start_kernel() self.kernel = self.manager.get_kernel(kernel_id) self.client = BlockingKernelClient() self.client.load_connection_file(self.kernel.connection_file) def Execute(self, request, context): response = self.client.execute_interactive( code=request.command, user_expressions={'test': request.expression}) expression = response['content']['user_expressions']['test']['data'] result = expression[ 'text/html'] if 'text/html' in expression else expression[ 'text/plain'] return pyexecutor_pb2.ExecuteResponse(result=result)
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready This function was taken from the ipykernel project. We plan to remove it when dropping support for python 2. Yields ------- client: jupyter_client.BlockingKernelClient connected to the kernel """ kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE) try: connection_file = os.path.join( paths.jupyter_runtime_dir(), 'kernel-%i.json' % kernel.pid, ) # wait for connection file to exist, timeout after 5s tic = time.time() while not os.path.exists(connection_file) \ and kernel.poll() is None \ and time.time() < tic + SETUP_TIMEOUT: time.sleep(0.1) if kernel.poll() is not None: o, e = kernel.communicate() if not PY3 and isinstance(e, bytes): e = e.decode() raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() client.wait_for_ready() try: yield client finally: client.stop_channels() finally: if not PY2: kernel.terminate()
def setup(): global client kernel = Popen([sys.executable, '-m', 'ipykernel'], stdout=PIPE, stderr=PIPE) connection_file = os.path.join( paths.jupyter_runtime_dir(), 'kernel-%i.json' % kernel.pid, ) sleep(1) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() client.wait_for_ready() loaded = client.execute_interactive(load_splonky) if loaded['content']['status'] == 'error': raise Exception("Could not load core Splonky libraries") os_process_id = re.findall('.*\/kernel-(\d+)\.json$', connection_file)[0] return os_process_id
def connect_kernel(cf): """ Connect a kernel. """ if is_runing(cf): kc = BlockingKernelClient(connection_file=cf) kc.load_connection_file(cf) km = None else: # Kernel manager km = manager.KernelManager(connection_file=cf) km.start_kernel() # Kernel Client kc = km.blocking_client() init_kernel(kc) return km, kc
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE, env=env) connection_file = os.path.join(IPYTHONDIR, 'profile_default', 'security', 'kernel-%i.json' % kernel.pid) # wait for connection file to exist, timeout after 5s tic = time.time() while not os.path.exists(connection_file) \ and kernel.poll() is None \ and time.time() < tic + SETUP_TIMEOUT: time.sleep(0.1) if kernel.poll() is not None: o, e = kernel.communicate() e = py3compat.cast_unicode(e) raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() client.wait_for_ready() try: yield client finally: client.stop_channels() kernel.terminate()
def execute_from_command_line(): if sys.argv.count('--existing') != 1: raise ValueError(f'{sys.argv}\n' f'--existing argument must occur once only.') kernel_arg_index = sys.argv.index('--existing') try: kernel_name = sys.argv[kernel_arg_index + 1] except IndexError: # Following the command-line API of jupyter console, qtconsole etc, the --existing argument # can be used without a value, meaning use the kernel whose connection file has most # recently been accessed. We support that here when --existing is the last element of the # command line. Otherwise, the behavior of the no-argument-value form can be achieved with # --existing ''. kernel_name = None else: sys.argv.pop(kernel_arg_index + 1) sys.argv.pop(kernel_arg_index) if {'shell', 'shell_plus'} & set(sys.argv): # Special case: use `jupyter console` for management commands requesting a python shell. argv = [ 'jupyter', 'console', '--Completer.use_jedi=False', '--existing' ] if kernel_name: argv.append(kernel_name) os.execlp(argv[0], *argv) connection_file = find_connection_file( kernel_name) if kernel_name else find_connection_file() kernel_client = BlockingKernelClient(connection_file=connection_file) kernel_client.load_connection_file() response = kernel_client.execute_interactive(f""" from devkernel.kernel import execute_from_command_line execute_from_command_line('{json.dumps(sys.argv)}') """) exit_status = 0 if response['metadata']['status'] == 'ok' else 1 sys.exit(exit_status)
def run(self): cf = find_connection_file() client = BlockingKernelClient(connection_file=cf) client.load_connection_file() client.start_channels(shell=False, iopub=True, stdin=False, control=True, hb=False) while True: try: msg = client.get_iopub_msg(TIMEOUT) self.pub_q.put(msg) except Empty: pass if self.cmd_q.qsize(): cmd = self.cmd_q.get() if cmd is None: print('Client thread closing') break client.execute(cmd) self.ctrl_q.put(client.get_shell_msg())
def JKConnect(self) -> None: runtime_dir = pathlib.Path(jupyter_core.paths.jupyter_runtime_dir()) connection_files = runtime_dir.glob("kernel-*.json") source = '\n'.join( connection_file.name.lstrip('kernel-').rstrip('.json') + ' ' + datetime.fromtimestamp(connection_file.stat().st_ctime).strftime( "%m/%d %H:%M") for connection_file in connection_files) proc = subprocess.run("fzf-tmux|awk '{print $1}'", input=source, stdout=PIPE, shell=True, text=True) connection_file = 'kernel-%s.json' % proc.stdout.strip() connection_file = runtime_dir.joinpath(connection_file).as_posix() kc = BlockingKernelClient() try: kc.load_connection_file(connection_file) kc.execute_interactive('', timeout=1) except (TimeoutError, FileNotFoundError): self.nvim.command("echoerr 'Selected connection is dead!'") else: self.executor = ExecutePreprocessor() self.executor.kc = kc self.nvim.command("echo 'Successfully connected!'")
class Kernel(object): def __init__(self, active_dir, pyspark): # kernel config is stored in a dot file with the active directory config = os.path.join(active_dir, ".kernel-%s.json" % str(uuid.uuid4())) # right now we're spawning a child process for IPython. we can # probably work directly with the IPython kernel API, but the docs # don't really explain how to do it. log_file = None if pyspark: os.environ["IPYTHON_OPTS"] = "kernel -f %s" % config pyspark = os.path.join(os.environ.get("SPARK_HOME"), "bin/pyspark") spark_log = os.environ.get("SPARK_LOG", None) if spark_log: log_file = open(spark_log, "w") spark_opts = os.environ.get("SPARK_OPTS", "") args = [pyspark] + spark_opts.split() # $SPARK_HOME/bin/pyspark <SPARK_OPTS> p = subprocess.Popen(args, stdout=log_file, stderr=log_file) else: args = [sys.executable, '-m', 'IPython', 'kernel', '-f', config] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # when __this__ process exits, we're going to remove the ipython config # file and kill the ipython subprocess atexit.register(p.terminate) def remove_config(): os.remove(config) atexit.register(remove_config) def close_file(): if log_file: log_file.close() atexit.register(close_file) # i found that if i tried to connect to the kernel immediately, it wasn't # up and running. 1.5 seconds was arbitrarily chosen (but seems to work) time.sleep(1.5) # fire up the kernel with the appropriate config self.client = BlockingKernelClient(connection_file=config) self.client.load_connection_file() self.client.start_channels() # load our monkeypatches... self.client.execute("%matplotlib inline") self.client.execute(autocomplete_patch) self.client.execute(vars_patch) def _run_code(self, code, timeout=0.1): # this function executes some code and waits for it to completely finish # before returning. i don't think that this is neccessarily the best # way to do this, but the IPython documentation isn't very helpful for # this particular topic. # # 1) execute code and grab the ID for that execution thread # 2) look for messages coming from the "iopub" channel (this is just a # stream of output) # 3) when we get a message that is one of the following, save relevant # data to `data`: # - execute_result - content from repr # - stream - content from stdout # - error - ansii encoded stacktrace # the final piece is that we check for when the message indicates that # the kernel is idle and the message's parent is the original execution # ID (msg_id) that's associated with our executing code. if this is the # case, we'll return the data and the msg_id and exit msg_id = self.client.execute(code) output = { "msg_id": msg_id, "output": None, "image": None, "error": None } while True: try: reply = self.client.get_iopub_msg(timeout=timeout) except Empty: continue if "execution_state" in reply['content']: if reply['content']['execution_state']=="idle" and reply['parent_header']['msg_id']==msg_id: if reply['parent_header']['msg_type']=="execute_request": return output elif reply['header']['msg_type']=="execute_result": output['output'] = reply['content']['data'].get('text/plain', '') elif reply['header']['msg_type']=="display_data": output['image'] = reply['content']['data'].get('image/png', '') elif reply['header']['msg_type']=="stream": output['output'] = reply['content'].get('text', '') elif reply['header']['msg_type']=="error": output['error'] = "\n".join(reply['content']['traceback']) def execute(self, code): return self._run_code(code) def complete(self, code): # i couldn't figure out how to get the autocomplete working with the # ipython kernel (i couldn't get a completion_reply from the iopub), so # we're using jedi to do the autocompletion. the __autocomplete is # defined in `autocomplete_patch` above. return self.execute("__autocomplete('%s')" % code) def get_dataframes(self): return self.execute("__get_variables()")
class ToreeClient: def __init__(self, connectionFileLocation): self.client = BlockingKernelClient( connection_file=connectionFileLocation) self.client.load_connection_file( connection_file=connectionFileLocation) def is_alive(self): return self.client.is_alive() def is_ready(self): try: result = self.eval('1') if result == '1': return True else: return False except: return False def wait_for_ready(self, timeout=TIMEOUT): # Wait for initialization, by receiving an 'idle' message # Flush Shell channel abs_timeout = time.time() + timeout while True: try: msg = self.client.shell_channel.get_msg(block=True, timeout=0.2) except Empty: break # Check if current time is ready check time plus timeout if time.time() > abs_timeout: raise RuntimeError("Kernel didn't respond in %d seconds" % timeout) # Flush IOPub channel while True: try: msg = self.client.iopub_channel.get_msg(block=True, timeout=0.2) except Empty: break # Check if current time is ready check time plus timeout if time.time() > abs_timeout: raise RuntimeError("Kernel didn't respond in %d seconds" % timeout) def eval(self, code, timeout=TIMEOUT): # Validate that remote kernel is available before submitting request if self.client.is_alive() == False: raise Exception( 'Problem connecting to remote kernel: Kernel is NOT alive') debug_print('-----------------------------------------') debug_print('Executing: ') debug_pprint(code) # submit request and retrieve the message id for the execution msg_id = self.client.execute(code=code, allow_stdin=False) debug_print('Message id for code execution:' + msg_id) # now the kernel should be 'busy' with [parent_header][msg_id] being the current message try: busy_msg = self.client.iopub_channel.get_msg(block=True, timeout=timeout) except: raise Exception('Error: Timeout retrieving busy status message') debug_print('Current kernel status (%s): %s' % (busy_msg['parent_header']['msg_id'], busy_msg['content']['execution_state'])) if busy_msg['content']['execution_state'] == 'busy': debug_print('busy_message received as expected') else: debug_print('Error: did not receive busy message for request %s' % msg_id) debug_pprint(busy_msg) # Check message reply status (ok / error) debug_print('Waiting for status reply') reply = self.client.get_shell_msg(block=True, timeout=timeout) debug_print('message reply: %s' % reply['content']['status']) debug_pprint(reply) type = '' results = [] while True: try: msg = self.client.get_iopub_msg(timeout=timeout) except: raise Exception("Error: Timeout executing request") debug_print('message') debug_pprint(msg) # validate that the responses are still related to current request if msg['parent_header']['msg_id'] != msg_id: debug_print('Warning: Invalid message id received ' + msg['parent_header']['msg_id'] + ' expected ' + msg_id) continue # validate execute_inputs are from current code elif msg['msg_type'] == 'execute_input': debug_print('current message status: ' + msg['msg_type']) debug_print('current message content code: ' + msg['content']['code']) if msg['content']['code'] == code: continue # Stream results are being returned, accumulate them to results elif msg['msg_type'] == 'stream': type = 'stream' results.append(msg['content']['text']) continue # Execute_Results are being returned: # They can be text/plain or text/html # accumulate them to results elif msg['msg_type'] == 'execute_result': debug_print('Received results of type: %s ' % msg['content']['data']) if 'text/plain' in msg['content']['data']: type = 'text' results.append(msg['content']['data']['text/plain']) elif 'text/html' in msg['content']['data']: type = 'html' results.append(msg['content']['data']['text/html']) continue # When idle, responses have all been processed/returned elif msg['msg_type'] == 'status': debug_print('current message status: ' + msg['content']['execution_state']) if msg['content']['execution_state'] == 'idle': break else: debug_print('Message ignored: %s' % msg['msg_type']) if reply['content']['status'] == 'ok': debug_print('Returning sucessful invocation result') if type == 'html': html = ''.join(results) htmlWrapper = HtmlOutput(html) return htmlWrapper else: return ''.join(results) else: debug_print('Returning failed invocation exception') error = '' if 'ename' in reply['content']: error = reply['content']['ename'] error_message = '' if 'evalue' in reply['content']: error_message = reply['content']['evalue'] raise Exception('Error: %s - %s' % (error, error_message))
class DaisyWorkflow_client: def __init__(self, connection_file=None, executable=False): import os self.alg_keys = [] self.dat_keys = [] #super().__init__() self.kc = BlockingKernelClient() if connection_file == None: raise Exception( 'Please Specific Connection file to Remote IPython Kernel first' ) if os.access(connection_file, os.R_OK) == False: raise Exception('The connection file can no be read!') self.kc.load_connection_file(connection_file) try: self.kc.start_channels() except RuntimeError: raise Exception( 'Can not start channels, Please CHECK REMOTE KERNEL STATUS') self.executable = executable self.remote_name = None self.alg_clients = {} def initialize(self, class_name=None, workflow_name=None, workflow_cfgfile=None, algorithms_cfgfile=None): import os, json if class_name == None: raise Exception('Please Specific Workflow class name first') cmd = "from Daisy.Workflow import " + class_name self.kc.execute_interactive(cmd) if workflow_name == None: workflow_name = class_name self.remote_name = workflow_name cmd = self.remote_name + " = " + class_name + "('" + workflow_name + "')" self.kc.execute_interactive(cmd) if workflow_cfgfile == None: raise Exception('Please Specific Workflow Config file first') if os.access(workflow_cfgfile, os.R_OK) == False: raise Exception('The Workflow Config file can no be read!') with open(workflow_cfgfile, 'r') as json_file: string = json_file.read() wf_cfg = json.loads(string) temp_name = 'cfg_dict' + str(randint(1, 1000000)) cmd = temp_name + ' = ' + str(wf_cfg) self.kc.execute_interactive(cmd) cmd = self.remote_name + ".initialize(workflow_engine='PyWorkflowEngine', workflow_environment = " + temp_name + ")" self.kc.execute_interactive(cmd) self.kc.execute_interactive('del ' + temp_name) def setLogLevel(self, level): pass #Sniper.setLogLevel(level) #super().setLogLevel(level) def data_keys(self): cmd = self.remote_name + ".data_keys()" msg_id = self.kc.execute(cmd) exe_msg = self.execute_status(msg_id) dat_keys = [] if 'data' in exe_msg: msg = exe_msg['data'] msg = msg[msg.find("[") + 1:msg.rfind("]")] items = msg.split(',') #items = exe_msg['data'].split('\n') for i in items: begin = i.find("'") end = i.rfind("'") dat_keys.append(i[begin + 1:end]) self.dat_keys = dat_keys return self.dat_keys def algorithm_keys(self): cmd = self.remote_name + ".algorithm_keys()" msg_id = self.kc.execute(cmd) exe_msg = self.execute_status(msg_id) alg_keys = [] if 'data' in exe_msg: msg = exe_msg['data'] msg = msg[msg.find("[") + 1:msg.rfind("]")] items = msg.split(',') for i in items: begin = i.find("'") end = i.rfind("'") alg_keys.append(i[begin + 1:end]) self.alg_keys = alg_keys return self.alg_keys def get_data(self, data_name): raise Exception('Cannot get DataObject from Server') #return self.engine.datastore[data_name] def get_algorithm(self, algorithm_name): if algorithm_name in self.alg_clients.keys(): return self.alg_clients[algorithm_name] cmd = self.remote_name if algorithm_name in self.alg_keys: cmd = cmd + ".get_algorithm('" + algorithm_name + "')" elif algorithm_name in self.algorithm_keys(): cmd = cmd + ".get_algorithm('" + algorithm_name + "')" else: return False msg_id = self.kc.execute(cmd) exe_msg = self.execute_status(msg_id) self.alg_clients[algorithm_name] = DaisyAlgorithm_client( self.kc, exe_msg['code']) print(exe_msg['data']) return self.alg_clients[algorithm_name] def execute(self): pass #raise Exception('Must') def finalize(self): cmd = self.remote_name cmd = cmd + ".finalize()" self.kc.execute_interactive(cmd) #exe_msg = self.execute_status(msg_id) #print(exe_msg) #self.engine.finalize() def execute_status(self, msg_id): code_flag = False data_flag = False exe_msg = {'msg_id': msg_id} while True: try: kc_msg = self.kc.get_iopub_msg(timeout=5) if 'parent_header' in kc_msg and kc_msg['parent_header'][ 'msg_id'] != exe_msg['msg_id']: continue #_output_hook_default(kc_msg) msg_type = kc_msg['header']['msg_type'] msg_cont = kc_msg['content'] if msg_type == 'stream': exe_msg[msg_cont['name']] = msg_cont['text'] elif msg_type == 'error': exe_msg['error'] = msg_cont['traceback'] print(msg_cont['traceback']) break elif msg_type in ('display_data', 'execute_result'): if 'data' in msg_cont: data_flag = True exe_msg['data'] = msg_cont['data'].get( 'text/plain', '') if 'code' in msg_cont: code_flag = True exe_msg['code'] = msg_cont['code'] if code_flag and data_flag: break except: print('timeout kc.get_iopub_msg') break return exe_msg
class SendToIPython(object): def __init__(self, nvim): self.nvim = nvim self.client = None self.kerneldir = Path(jupyter_runtime_dir()) @neovim.function('RunningKernels', sync=True) def running_kernels(self, args): l = self.kerneldir.glob('kernel-*.json') l = sorted(l, reverse=True, key=lambda f: f.stat().st_ctime) return [f.name for f in l] @neovim.command('SendTo', complete='customlist,RunningKernels', nargs='?') def send_to(self, args): cfs = args or self.running_kernels(None) if not cfs: self.nvim.command('echom "No kernel found"') return if self.client is not None: self.client.stop_channels() cf = cfs[0] self.client = BlockingKernelClient() self.client.load_connection_file(self.kerneldir / cf) self.client.start_channels() # run function once to register it for the `funcref` function self.nvim.command('call SendLinesToJupyter()') self.nvim.command( 'let g:send_target = {"send": funcref("SendLinesToJupyter")}') self.nvim.command('echom "Sending to %s"' % cf) @neovim.function('SendLinesToJupyter') def send_lines(self, args): if args: self.client.execute('\n'.join(args[0])) @neovim.function('SendComplete', sync=True) def complete(self, args): findstart, base = args if self.client is None: return -3 # no client setup yet: cancel silently and leave completion mode if findstart: line = self.nvim.current.line if not line: return -2 # empty line: cancel silently but stay in completion mode pos = self.nvim.current.window.cursor[1] try: reply = self.client.complete(line, pos, reply=True, timeout=timeout)['content'] except TimeoutError: return -2 self.completions = [{ 'word': w, 'info': ' ' } for w in reply['matches']] return reply['cursor_start'] else: # TODO: use vim's complete_add/complete_check for async operation get_info(self.client, self.completions) return {'words': self.completions, 'refresh': 'always'} @neovim.function('SendCanComplete', sync=True) def can_complete(self, args): return args[ 0] != '' and self.client is not None and self.client.is_alive()
class Kernel(object): def __init__(self, active_dir): # kernel config is stored in a temp file config = os.path.join(active_dir, ".kernel-%s.json" % str(uuid.uuid4())) args = [sys.executable, '-m', 'IPython', 'kernel', '-f', config] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # when __this__ process exits, we're going to remove the ipython config # file and kill the ipython subprocess atexit.register(p.terminate) def remove_config(): os.remove(config) atexit.register(remove_config) # i found that if i tried to connect to the kernel immediately, it wasn't up # and running. 1.5 seconds was arbitrarily chosen (but seems to work) time.sleep(1.5) # fire up the kernel with the appropriate config self.client = BlockingKernelClient(connection_file=config) self.client.load_connection_file() self.client.start_channels() # load our monkeypatches... self.client.execute("%matplotlib inline") self.client.execute(autocomplete_patch) self.client.execute(vars_patch) def _run_code(self, code, timeout=0.1): # this function executes some code and waits for it to completely finish before # returning. i don't think that this is neccessarily the best way to do this, but # the IPython documentation isn't very helpful for this particular topic. # # 1) execute code and grab the ID for that execution thread # 2) look for messages coming from the "iopub" channel (this is just a stream of output) # 3) when we get a message that is one of the following, save relevant data to `data`: # - execute_result - content from repr # - stream - content from stdout # - error - ansii encoded stacktrace # the final piece is that we check for when the message indicates that the kernel is idle # and the message's parent is the original execution ID (msg_id) that's associated with # our executing code. if this is the case, we'll return the data and the msg_id and exit msg_id = self.client.execute(code) data = None image = None while True: try: reply = self.client.get_iopub_msg(timeout=timeout) except Empty: continue if "execution_state" in reply['content']: if reply['content']['execution_state']=="idle" and reply['parent_header']['msg_id']==msg_id: if reply['parent_header']['msg_type']=="execute_request": return { "msg_id": msg_id, "output": data, "image": image } elif reply['header']['msg_type']=="execute_result": data = reply['content']['data'].get('text/plain', '') elif reply['header']['msg_type']=="display_data": image = reply['content']['data'].get('image/png', '') elif reply['header']['msg_type']=="stream": data = reply['content'].get('text', '') elif reply['header']['msg_type']=="error": data = "\n".join(reply['content']['traceback']) def execute(self, code): return self._run_code(code) def complete(self, code): # i couldn't figure out how to get the autocomplete working with the ipython # kernel (i couldn't get a completion_reply from the iopub), so we're using # jedi to do the autocompletion. the __autocomplete is defined in `autocomplete_patch` # above. return self.execute("__autocomplete('%s')" % code)
'OKGREEN': '\033[92m', 'WARNING': '\033[93m', 'FAIL': '\033[91m', 'ENDC': '\033[0m', 'BOLD': '\033[1m', 'UNDERLINE': '\033[4m', } print(lookup[color], end='') print(*args, end='') print(lookup['ENDC']) # setup by automatically finding a running kernel cf = find_connection_file() client = BlockingKernelClient(connection_file=cf) client.load_connection_file() client.start_channels() # simplest usage - execute statments and check if OK msgid = client.execute('a = 2') ret = client.get_shell_msg() status = ret['content']['status'] if status == 'ok': print('statement executed ok') elif status == 'error': ename = ret['content']['ename'] print('there was a %s exception, which will also appear on the ' 'iopub channel' % ename) # listen to what's going on in the kernel with blocking calls, # and take different actions depending on what's arriving
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ def connection_file_ready(connection_file): """Check if connection_file is a readable json file.""" if not os.path.exists(connection_file): return False try: with open(connection_file) as f: json.load(f) return True except ValueError: return False kernel = Popen([sys.executable, "-c", cmd], stdout=PIPE, stderr=PIPE, encoding="utf-8") try: connection_file = os.path.join( paths.jupyter_runtime_dir(), "kernel-%i.json" % kernel.pid, ) # wait for connection file to exist, timeout after 5s tic = time.time() while ( not connection_file_ready(connection_file) and kernel.poll() is None and time.time() < tic + SETUP_TIMEOUT ): time.sleep(0.1) # Wait 100ms for the writing to finish time.sleep(0.1) if kernel.poll() is not None: o, e = kernel.communicate() raise OSError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise OSError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() client.start_channels() client.wait_for_ready() try: yield client finally: client.stop_channels() finally: kernel.terminate() kernel.wait() # Make sure all the fds get closed. for attr in ["stdout", "stderr", "stdin"]: fid = getattr(kernel, attr) if fid: fid.close()
def main(kid, var, pid): # load connection info and init communication cf = find_connection_file(kid) # str(port)) km = BlockingKernelClient(connection_file=cf) km.load_connection_file() km.start_channels() # Step 0: get all the inputs load_input_code = f""" proc_id="{pid}" var={var} var_name="{var}" sql_name = "{cfg.sql_name}" sql_password = "******" sql_dbname = "{cfg.sql_dbname}" sql_schema_name = "{cfg.sql_schema_name}" sql_table_name = "{cfg.sql_table_name}" json_file_name = "/Users/peterchan/Desktop/GitHub/jupyter-extension/juneau_extension/data_file.json" """ # Step 1: access the table and convert it to JSON request_var_code = f""" import numpy as np import pandas as pd import json import copy if type(var) is pd.DataFrame or type(var) is np.ndarray or type(var) is list: df_json_string = var.to_json(orient='split', index=False) df_ls = json.loads(df_json_string)['data'] df_ls_copy = copy.deepcopy(df_ls) """ # Step 2: define the functions used to write to the JSON file json_lock_code = """ def initialize(): data = { "ownerID": "", "id123": "operating", "id124": "finish" } with open("./juneau_extension/data_file.json", "w") as file: json.dump(data, file, indent=4) def acquire_lock(pid): with open(json_file_name, "r+") as file: try: data = json.load(file) if data["ownerID"]: return False else: file.seek(0) file.truncate() data['ownerID'] = pid json.dump(data, file, indent=4) return True except Exception: return False def release_lock(pid): with open(json_file_name, "r+") as file: data = json.load(file) if data['ownerID'] == pid: file.seek(0) file.truncate() data['ownerID'] = "" json.dump(data, file, indent=4) # input: id of the process # remove from the file if the process is completed/ terminated/ timed out def update_exec_status(status, pid): done = False while not done: success = acquire_lock(pid) if success: try: with open(json_file_name, "r+") as file: data = json.load(file) if not data['ownerID'] == pid: continue file.seek(0) file.truncate() data[pid] = status json.dump(data, file, indent=4) release_lock(pid) done = True except Exception: continue return True """ # Step 3: connect to SQL and insert the table insert_code = """ from sqlalchemy import create_engine conn_string = f"postgresql://{sql_name}:{sql_password}@localhost/{sql_dbname}" table_string = f"{sql_schema_name}.{sql_table_name}" engine = create_engine(conn_string) with engine.connect() as connection: insertion_string = f'CREATE TABLE {sql_schema_name}.{var_name} ("A" int, "B" int, "C" int, "D" int);' for ls in df_ls_copy: insertion_string += f"INSERT INTO {sql_schema_name}.{var_name} VALUES ({ls[0]}, {ls[1]}, {ls[2]}, {ls[3]});" connection.execute(insertion_string) print(proc_id) update_exec_status("done", proc_id) rows = connection.execute(f"select * from {sql_schema_name}.{var_name} limit 5;") for row in rows: print(row) """ code = load_input_code + request_var_code + json_lock_code + insert_code km.execute_interactive(code, timeout=TIMEOUT) km.stop_channels()
class Kernel(object): # kernel config is stored in a dot file with the active directory def __init__(self, config, active_dir, pyspark): # right now we're spawning a child process for IPython. we can # probably work directly with the IPython kernel API, but the docs # don't really explain how to do it. log_file = None if pyspark: os.environ["IPYTHON_OPTS"] = "kernel -f %s" % config pyspark = os.path.join(os.environ.get("SPARK_HOME"), "bin/pyspark") spark_log = os.environ.get("SPARK_LOG", None) if spark_log: log_file = open(spark_log, "w") spark_opts = os.environ.get("SPARK_OPTS", "") args = [pyspark] + spark_opts.split() # $SPARK_HOME/bin/pyspark <SPARK_OPTS> p = subprocess.Popen(args, stdout=log_file, stderr=log_file) else: args = [sys.executable, '-m', 'IPython', 'kernel', '-f', config] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # when __this__ process exits, we're going to remove the ipython config # file and kill the ipython subprocess atexit.register(p.terminate) def remove_config(): if os.path.isfile(config): os.remove(config) atexit.register(remove_config) # i found that if i tried to connect to the kernel immediately, so we'll # wait until the config file exists before moving on while os.path.isfile(config)==False: time.sleep(0.1) def close_file(): if log_file: log_file.close() atexit.register(close_file) # fire up the kernel with the appropriate config self.client = BlockingKernelClient(connection_file=config) self.client.load_connection_file() self.client.start_channels() # load our monkeypatches... self.client.execute("%matplotlib inline") python_patch_file = os.path.join(dirname, "langs", "python-patch.py") self.client.execute("%run " + python_patch_file) def _run_code(self, execution_id, code, timeout=0.1): # this function executes some code and waits for it to completely finish # before returning. i don't think that this is neccessarily the best # way to do this, but the IPython documentation isn't very helpful for # this particular topic. # # 1) execute code and grab the ID for that execution thread # 2) look for messages coming from the "iopub" channel (this is just a # stream of output) # 3) when we get a message that is one of the following, save relevant # data to `data`: # - execute_result - content from repr # - stream - content from stdout # - error - ansii encoded stacktrace # the final piece is that we check for when the message indicates that # the kernel is idle and the message's parent is the original execution # ID (msg_id) that's associated with our executing code. if this is the # case, we'll return the data and the msg_id and exit msg_id = self.client.execute(code, allow_stdin=False) request = { "id": execution_id, "msg_id": msg_id, "code": code, "status": "started" } sys.stdout.write(json.dumps(request) + '\n') sys.stdout.flush() output = { "id": execution_id, "msg_id": msg_id, "output": "", "stream": None, "image": None, "error": None } while True: try: reply = self.client.get_iopub_msg(timeout=timeout) except Empty: continue if "execution_state" in reply['content']: if reply['content']['execution_state']=="idle" and reply['parent_header']['msg_id']==msg_id: if reply['parent_header']['msg_type']=="execute_request": request["status"] = "complete" sys.stdout.write(json.dumps(request) + '\n') sys.stdout.flush() return elif reply['header']['msg_type']=="execute_result": output['output'] = reply['content']['data'].get('text/plain', '') output['stream'] = reply['content']['data'].get('text/plain', '') elif reply['header']['msg_type']=="display_data": if 'image/png' in reply['content']['data']: output['image'] = reply['content']['data']['image/png'] elif 'text/html' in reply['content']['data']: output['html'] = reply['content']['data']['text/html'] elif reply['header']['msg_type']=="stream": output['output'] += reply['content'].get('text', '') output['stream'] = reply['content'].get('text', '') elif reply['header']['msg_type']=="error": output['error'] = "\n".join(reply['content']['traceback']) # TODO: if we have something non-trivial to send back... sys.stdout.write(json.dumps(output) + '\n') sys.stdout.flush() # TODO: should probably get rid of all this output['stream'] = None output['image'] = None output['html'] = None def _complete(self, execution_id, code, timeout=0.5): # Call ipython kernel complete, wait for response with the correct msg_id, # and construct appropriate UI payload. # See below for an example response from ipython kernel completion for 'el' # # { # 'parent_header': # {u'username': u'ubuntu', u'version': u'5.0', u'msg_type': u'complete_request', # u'msg_id': u'5222d158-ada8-474e-88d8-8907eb7cc74c', u'session': u'cda4a03d-a8a1-4e6c-acd0-de62d169772e', # u'date': datetime.datetime(2015, 5, 7, 15, 25, 8, 796886)}, # 'msg_type': u'complete_reply', # 'msg_id': u'a3a957d6-5865-4c6f-a0b2-9aa8da718b0d', # 'content': # {u'matches': [u'elif', u'else'], u'status': u'ok', u'cursor_start': 0, u'cursor_end': 2, u'metadata': {}}, # 'header': # {u'username': u'ubuntu', u'version': u'5.0', u'msg_type': u'complete_reply', # u'msg_id': u'a3a957d6-5865-4c6f-a0b2-9aa8da718b0d', u'session': u'f1491112-7234-4782-8601-b4fb2697a2f6', # u'date': datetime.datetime(2015, 5, 7, 15, 25, 8, 803470)}, # 'buffers': [], # 'metadata': {} # } # msg_id = self.client.complete(code) request = { "id": execution_id, "msg_id": msg_id, "code": code, "status": "started" } sys.stdout.write(json.dumps(request) + '\n') sys.stdout.flush() output = { "id": execution_id, "msg_id": msg_id, "output": None, "image": None, "error": None } while True: try: reply = self.client.get_shell_msg(timeout=timeout) except Empty: continue if "matches" in reply['content'] and reply['msg_type']=="complete_reply" and reply['parent_header']['msg_id']==msg_id: results = [] for completion in reply['content']['matches']: result = { "value": completion, "dtype": "---" } if "." in code: # result['text'] = result['value'] # ".".join(result['value'].split(".")[1:]) result['text'] = result['value'] #.split('.')[-1] result["dtype"] = "function" else: result['text'] = result['value'] result["dtype"] = "" # type(globals().get(code)).__name__ results.append(result) output['output'] = results output['status'] = "complete" sys.stdout.write(json.dumps(output) + '\n') sys.stdout.flush() return def execute(self, execution_id, code, complete=False): if complete==True: return self._complete(execution_id, code) else: result = self._run_code(execution_id, code) if re.match("%?reset", code): # load our monkeypatches... k.client.execute("%matplotlib inline") k.client.execute(vars_patch) return result def get_packages(self): return self.execute("__get_packages()")
def main(): client = BlockingKernelClient() client.load_connection_file(find_connection_file()) code = "; ".join(sys.argv[1:]) if len(sys.argv) > 1 else sys.stdin.read() client.execute(code, allow_stdin=False)
import time from queue import Empty # Py 3 from jupyter_client import connect from jupyter_client import BlockingKernelClient logging.basicConfig(level=logging.DEBUG) f = connect.find_connection_file() print("loaded") client = BlockingKernelClient() client.load_connection_file(f) print("prepared") # msg_id = client.execute("print('hello')") msg_id = client.execute("1 + 10") # client.wait_for_ready() res = client.get_shell_msg(msg_id, timeout=1) print(res) print("----------------------------------------") msg = res["msg_id"] for i in range(10): if not client.is_alive():
def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready Returns ------- kernel_manager: connected KernelManager instance """ def connection_file_ready(connection_file): """Check if connection_file is a readable json file.""" if not os.path.exists(connection_file): return False try: with open(connection_file) as f: json.load(f) return True except ValueError: return False kernel = Popen([sys.executable, '-c', cmd], stdout=PIPE, stderr=PIPE) try: # connection_file = os.path.join( # paths.jupyter_runtime_dir(), # 'kernel-%i.json' % kernel.pid, # ) connection_file = "./remotekernel2.json" # # wait for connection file to exist, timeout after 5s # tic = time.time() # while not connection_file_ready(connection_file) \ # and kernel.poll() is None \ # and time.time() < tic + SETUP_TIMEOUT: # time.sleep(0.1) # # # Wait 100ms for the writing to finish # time.sleep(0.1) # # if kernel.poll() is not None: # o,e = kernel.communicate() # e = py3compat.cast_unicode(e) # raise IOError("Kernel failed to start:\n%s" % e) # # if not os.path.exists(connection_file): # if kernel.poll() is None: # kernel.terminate() # raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) client.load_connection_file() # client.start_channels() # client.wait_for_ready() # client. comm_manager = CommManager(parent=client, kernel_client=client) comm = comm_manager.new_comm("neos_comm") # comm.open() nep = Nep(comm, kernel_id=client.kernel_info()) nep.start() # self.nep_start(comm) # print(client.) try: yield client finally: # client.stop_channels() pass finally: # kernel.terminate() pass
class Kernel(object): def __init__(self, active_dir): # kernel config is stored in a dot file with the active directory config = os.path.join(active_dir, ".kernel-%s.json" % str(uuid.uuid4())) # right now we're spawning a child process for IPython. we can # probably work directly with the IPython kernel API, but the docs # don't really explain how to do it. args = [sys.executable, '-m', 'IPython', 'kernel', '-f', config] p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # when __this__ process exits, we're going to remove the ipython config # file and kill the ipython subprocess atexit.register(p.terminate) def remove_config(): os.remove(config) atexit.register(remove_config) # i found that if i tried to connect to the kernel immediately, it wasn't # up and running. 1.5 seconds was arbitrarily chosen (but seems to work) time.sleep(1.5) # fire up the kernel with the appropriate config self.client = BlockingKernelClient(connection_file=config) self.client.load_connection_file() self.client.start_channels() # load our monkeypatches... self.client.execute("%matplotlib inline") self.client.execute(autocomplete_patch) self.client.execute(vars_patch) def _run_code(self, code, timeout=0.1): # this function executes some code and waits for it to completely finish # before returning. i don't think that this is neccessarily the best # way to do this, but the IPython documentation isn't very helpful for # this particular topic. # # 1) execute code and grab the ID for that execution thread # 2) look for messages coming from the "iopub" channel (this is just a # stream of output) # 3) when we get a message that is one of the following, save relevant # data to `data`: # - execute_result - content from repr # - stream - content from stdout # - error - ansii encoded stacktrace # the final piece is that we check for when the message indicates that # the kernel is idle and the message's parent is the original execution # ID (msg_id) that's associated with our executing code. if this is the # case, we'll return the data and the msg_id and exit msg_id = self.client.execute(code) data = None image = None while True: try: reply = self.client.get_iopub_msg(timeout=timeout) except Empty: continue if "execution_state" in reply['content']: if reply['content']['execution_state'] == "idle" and reply[ 'parent_header']['msg_id'] == msg_id: if reply['parent_header']['msg_type'] == "execute_request": return { "msg_id": msg_id, "output": data, "image": image } elif reply['header']['msg_type'] == "execute_result": data = reply['content']['data'].get('text/plain', '') elif reply['header']['msg_type'] == "display_data": image = reply['content']['data'].get('image/png', '') elif reply['header']['msg_type'] == "stream": data = reply['content'].get('text', '') elif reply['header']['msg_type'] == "error": data = "\n".join(reply['content']['traceback']) def execute(self, code): return self._run_code(code) def complete(self, code): # i couldn't figure out how to get the autocomplete working with the # ipython kernel (i couldn't get a completion_reply from the iopub), so # we're using jedi to do the autocompletion. the __autocomplete is # defined in `autocomplete_patch` above. return self.execute("__autocomplete('%s')" % code) def get_dataframes(self): return self.execute("__get_variables()")