def __init__(self, **kwargs): super().__init__(**kwargs) self._eventloop = IOLoop.current() self._key = generate_id(size=32).encode('utf-8') self._execution_lock = Lock(loop=self._eventloop.asyncio_loop) self._inner_state_lock = Lock(loop=self._eventloop.asyncio_loop) self._dep_tracker = DependencyTracker() self._exec_unit_container = ExecUnitContainer() self.formatter = DisplayFormatter() self.manager_ns = BuiltInManager() self.manager_ns.global_ns['__builtins__'] = builtin_mods.__dict__ self.manager_ns.global_ns['show_graph'] = self._show_graph self._execution_ctx = Executor(self._exec_unit_container, manager_ns=self.manager_ns) self.KernelTB = ultratb.AutoFormattedTB(mode='Plain', color_scheme='LightBG', tb_offset=1, debugger_cls=None) self._execution_queue = Queue(loop=self._eventloop.asyncio_loop) builtin_mods.show_graph = self._show_graph # mapping from variable name (target id) to (request id, generator # object) self._registered_generators = dict() self._eventloop.spawn_callback(self._execution_loop)
def postprocess(cell_outputs, saved_cell_outputs, verbose=True, cell_n=None): """Unify outputs and saved_outputs to make them comparable. Side effects: 1) Print outputs 2) Reinitiate logging object """ if saved_cell_outputs is None: saved_cell_outputs = defaultdict(lambda: None) cell_outputs["stderr"], cell_outputs["stdout"] = ( cell_outputs["stderr"].getvalue(), cell_outputs["stdout"].getvalue(), ) if verbose: print( "-----> stderr of Cell %d:" % cell_n, cell_outputs["stderr"], "-----> stdout of Cell %d:" % cell_n, cell_outputs["stdout"], sep="\n", ) root_logger = logging.getLogger() # Reinitiate the logging object for handl in root_logger.handlers: root_logger.removeHandler(handl) logging.basicConfig() if saved_cell_outputs["stdout"] is None: saved_cell_outputs["stdout"] = "" if saved_cell_outputs["stderr"] is None: saved_cell_outputs["stderr"] = "" cell_outputs["stderr"] = escape_ansi(cell_outputs["stderr"]) if cell_outputs["display_data"] is None: cell_outputs["display_data"] = "" if saved_cell_outputs["display_data"] is None: saved_cell_outputs["display_data"] = "''" # elif isinstance(cell_outputs, str): # cell_outputs = "'" + cell_outputs + "'" celltest_disp = DisplayFormatter() cell_outputs["display_data"] = celltest_disp.format( cell_outputs["display_data"], include="text/plain")[0]["text/plain"] return cell_outputs, saved_cell_outputs
def init_display_formatter(self): if INLINE_OUTPUT_SUPPORTED: self.display_formatter = DisplayFormatter(parent=self) self.configurables.append(self.display_formatter) self.display_formatter.ipython_display_formatter.enabled = True else: super(PyDevTerminalInteractiveShell, self).init_display_formatter()
def _acquire_formatter(): try: # If the user is in ipython, get their configured display_formatter ip = get_ipython() # NOQA return ip.display_formatter except NameError: # otherwise use the default return DisplayFormatter()
def __init__(self, **kwargs): super().__init__(**kwargs) self._key = generate_id(size=32).encode('utf-8') self._eventloop = IOLoop.current().asyncio_loop self._inner_state_lock = Lock(loop=self._eventloop) self._dep_tracker = DependencyTracker() self._exec_unit_container = ExecUnitContainer() self.formatter = DisplayFormatter() self.ns_manager = BuiltInManager() self.initialize_builtins() self._execution_ctx = Executor(self._exec_unit_container, ns_manager=self.ns_manager) self.KernelTB = ultratb.AutoFormattedTB(mode='Plain', color_scheme='LightBG', tb_offset=1, debugger_cls=None) # mapping from variable name (target id) to (request id, generator # object) self._registered_generators = dict()
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Ensure the kernel we work with uses Futures on recv, so we can await them self.context = ctx = Context.instance() # Our subscription to messages from the kernel we launch self.iosub = ctx.socket(zmq.SUB) self.iosub.subscribe = b"" # From kernelapp.py, shell_streams are typically shell_stream, control_stream self.shell_stream = self.shell_streams[0] # Start with no child kernel self.child_kernel = None self.kernel_config = None self.acquiring_kernel = asyncio.Lock() self.kernel_launched = asyncio.Event() self.display_formatter = DisplayFormatter()
class ReactivePythonKernel(Kernel): implementation = 'reactivepy' implementation_version = __version__ language_info = { 'name': 'python', 'version': sys.version.split()[0], 'nbconvert_exporter': 'python', 'mimetype': 'text/x-python', 'file_extension': '.py' } banner = '' # measured in seconds. This value is intended to delay the same amount as # 60fps does between frames REGULAR_GENERATOR_DELAY = 0.016666 def __init__(self, **kwargs): super().__init__(**kwargs) self._eventloop = IOLoop.current() self._key = generate_id(size=32).encode('utf-8') self._execution_lock = Lock(loop=self._eventloop.asyncio_loop) self._inner_state_lock = Lock(loop=self._eventloop.asyncio_loop) self._dep_tracker = DependencyTracker() self._exec_unit_container = ExecUnitContainer() self.formatter = DisplayFormatter() self.ns_manager = BuiltInManager() self.initialize_builtins() self._execution_ctx = Executor(self._exec_unit_container, ns_manager=self.ns_manager) self.KernelTB = ultratb.AutoFormattedTB(mode='Plain', color_scheme='LightBG', tb_offset=1, debugger_cls=None) self._execution_queue = Queue(loop=self._eventloop.asyncio_loop) self._registered_generators = dict() self._eventloop.spawn_callback(self._execution_loop) def initialize_builtins(self): self.ns_manager.add_builtin('var_dependency_graph', self._var_dependency_graph) self.ns_manager.add_builtin('cell_dependency_graph', self._cell_dependency_graph) def _var_dependency_graph(self): h = Digraph() for start_node in self._dep_tracker.get_nodes(): if "-" in start_node: for dest_node in self._dep_tracker.get_neighbors(start_node): if "-" in dest_node: first = start_node[:start_node.find('-')] second = dest_node[:dest_node.find('-')] for char in "[]": first = first.replace(char, "") second = second.replace(char, "") h.edge(first, second) return h def _cell_dependency_graph(self): h = Digraph() for start_node in self._dep_tracker.get_nodes(): if "-" in start_node: for dest_node in self._dep_tracker.get_neighbors(start_node): if "-" in dest_node: first = self._exec_unit_container.get_by_display_id( start_node).pinning_cell second = self._exec_unit_container.get_by_display_id( dest_node).pinning_cell h.edge(first, second) return h async def _run_single_async_iter_step(self, item, exec_unit): # Make sure only single variable is important for execution # unit assert len(exec_unit.code_obj.output_vars) == 1 target_id = list(exec_unit.code_obj.output_vars)[0].get_name() request_id, async_gen_obj, insert_id = self._registered_generators[ target_id] async with self._inner_state_lock: if request_id != item.request.msg_id and insert_id < item.id: # self._log(item.request, # f"Cancelling ({request_id}, {insert_id}) for ({item.request.msg_id}, {item.id}) 1") await async_gen_obj.aclose() self._registered_generators[ target_id] = item.request.msg_id, item.async_gen_obj, item.id elif insert_id > item.id: return None try: exec_result = await self._execution_ctx.run_coroutine( anext(item.async_gen_obj), target_id, nohandle_exceptions=(StopAsyncIteration, )) except StopAsyncIteration as e: # iteration is complete raise e # Schedule next instance of iterator # Compute execution units which depend on this unit, # and schedule a new block to run which contains them descendants = self._dep_tracker.get_descendants(exec_unit.display_id) descendant_exec_units = list( map( lambda display_id: self._exec_unit_container.get_by_display_id( display_id), descendants)) next_iter_round = ExecBlock(item.request, exec_unit, descendant_exec_units, async_gen_obj=item.async_gen_obj) await self._execution_queue.put(next_iter_round) return exec_result async def _first_async_iter_step(self, item: ExecBlock, exec_unit: ExecutionUnitInfo, exec_result: ExecutionResult): # self._log(item.request, "Before inner state lock") async with self._inner_state_lock: # self._log(item.request, "After inner state lock") if exec_result.target_id in self._registered_generators: request_id, async_gen_obj, insert_id = self._registered_generators[ exec_result.target_id] if insert_id < item.id: # self._log( # item.request, f"Cancelling ({request_id}, {insert_id}) for ({item.request.msg_id}, {item.id}) 2") await async_gen_obj.aclose() else: raise ValueError(f"Very confuse. {insert_id} vs {item.id}") # self._log( # item.request, # f"Storing {(item.request.msg_id, exec_result.output, item.id)}") self._registered_generators[ exec_result. target_id] = item.request.msg_id, exec_result.output, item.id # Get first value and replace async gen with first # value or None async_gen_obj = exec_result.output try: exec_result.output = await anext(exec_result.output) except Exception: # DO not schedule further updates on failure return False # Schedule next instance of iterator # Compute execution units which depend on this unit, # and schedule a new block to run which contains # them descendants = self._dep_tracker.get_descendants(exec_unit.display_id) descendant_exec_units = list( map( lambda display_id: self._exec_unit_container.get_by_display_id( display_id), descendants)) next_iter_round = ExecBlock(item.request, exec_unit, descendant_exec_units, async_gen_obj=async_gen_obj) await self._execution_queue.put(next_iter_round) return True async def _execution_loop(self): while True: item: ExecBlock = await self._execution_queue.get() # self._log(item.request, f"Handling {item.request.msg_id}") if item is None: # Use None to signal end # self._log(item.request, 'Breaking execution loop!') break for exec_unit in ([item.current] + item.descendants): # async with self._execution_lock: # self._log(item.request, exec_unit.display_id) is_current = exec_unit == item.current is_iter = item.is_iter_exec and is_current if is_iter: # self._log(item.request, "Stepping iter") try: exec_result = await self._run_single_async_iter_step( item, exec_unit) except StopAsyncIteration: # self._log(item.request, # 'Step failed, breaking exec unit loop!') break else: # self._log(item.request, "Executing normally") # Execute code normally exec_result = self._execution_ctx.run_cell( exec_unit.code_obj.code, exec_unit.display_id) if exec_result.output is not None and not is_iter: is_awaitable, is_gen, is_async_gen = inspect_output_attrs( exec_result.output) # self._log(item.request, # f"{is_awaitable} {is_gen} {is_async_gen}") # These flags should be disjoint (never multiple True at same time) # but this layout allows for transforming a regular generator into an async generator # with a small timeout, and then passing it to the # async_gen branch # If the output is awaitable, wait for it and then replace # the old output if is_awaitable: exec_result.output = await exec_result.output # If the output is a regular generator, wrap it in an async # generator that will add a very small delay if is_gen: exec_result.output = convert_gen_to_async( exec_result.output, ReactivePythonKernel.REGULAR_GENERATOR_DELAY)() is_async_gen = True is_gen = False # If it is an async generator, either replace the old # generator that was active for this target name or # register this completely new generator. Remove the old # generator by canceling it if is_async_gen: # self._log(item.request, "Is now async gen!") setup_succeeded = await self._first_async_iter_step( item, exec_unit, exec_result) if not setup_succeeded: break if is_awaitable or is_gen or is_async_gen: self._execution_ctx.update_ns( {exec_result.target_id: exec_result.output}) if not item.request.silent: self._output_exec_results(exec_unit, item.request, is_current and not is_iter, exec_result) self._execution_queue.task_done() def _update_existing_exec_unit(self, code_obj, cell_id): # 4a. If it is a redefinition, get the old execution unit current_exec_unit = self._exec_unit_container.get_by_display_id( code_obj.display_id) if current_exec_unit.is_pinned and current_exec_unit.pinning_cell != cell_id: raise RedefiningOwnedCellException old_code_obj = current_exec_unit.code_obj new_input_vars = set(code_obj.input_vars) old_input_vars = set(old_code_obj.input_vars) # 4b. Compute the sets of edges to be added and deleted if the # variable dependencies of the cell have changed to_delete = old_input_vars - new_input_vars to_add = new_input_vars - old_input_vars # 4c. Actually replace old code object, delete old edges, # add new edges exec_unit = self._exec_unit_container.get_by_display_id( code_obj.display_id) exec_unit.code_obj = code_obj for sym in to_delete: code_object_id = self._exec_unit_container.get_by_symbol( sym).code_obj.display_id self._dep_tracker.delete_edge(code_object_id, code_obj.display_id) for sym in to_add: code_object_id = self._exec_unit_container.get_by_symbol( sym).code_obj.display_id self._dep_tracker.add_edge(code_object_id, code_obj.display_id) # 4d. Get the updated execution unit and return it return self._exec_unit_container.get_by_display_id(code_obj.display_id) def _create_new_exec_unit(self, code_obj, cell_id): # 4a. Create new execution unit to hold code object + display # id + cell id data current_exec_unit = self._exec_unit_container.register( ExecutionUnitInfo(code_obj, pinning_cell=cell_id)) # 4b. Add new node self._dep_tracker.add_node(code_obj.display_id) # 4c. For each variable it depends on, find the complete set of # defining variables (all the variables that were defined in # the same code block), and create a dependency to them for sym in code_obj.input_vars: dep = self._exec_unit_container.get_by_symbol(sym) if dep is not None: code_object_id = dep.code_obj.display_id self._dep_tracker.add_edge(code_object_id, code_obj.display_id) else: raise DefinitionNotFoundException(str(sym)) return current_exec_unit async def _inner_execute_request_callback(self, stream, ident, parent): # COPIED FROM IPYKERNEL/KERNELBASE.PY # async with self._execution_lock: try: request = RequestInfo(parent, ident) except BaseException: self.log.error(f"Got bad msg: {parent}") return # Re-broadcast our input for the benefit of listening clients, and # start computing output if not request.silent: request.execution_count = self.execution_count = 1 + self.execution_count self._publish_execute_input(request.code, parent, request.execution_count) stop_on_error = request.content.get('stop_on_error', True) response_meta = self.init_metadata(parent) reply_content = await self.do_execute(request) # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) response_meta = self.finish_metadata(parent, response_meta, reply_content) reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=response_meta, ident=ident) self.log.debug(f"{reply_msg}") if not request.silent and reply_msg['content'][ 'status'] == u'error' and stop_on_error: self._abort_queues() def execute_request(self, stream, ident, parent): """handle an execute_request""" self._eventloop.spawn_callback(self._inner_execute_request_callback, stream, ident, parent) def _update_kernel_state(self, code_obj, cell_id, deleted_cell_ids): # 3. If deletedCells was passed, then unpin all execution units # that were previously attached to a cell if deleted_cell_ids is not None: for to_delete_cell_id in deleted_cell_ids: self._exec_unit_container.unpin_exec_unit(to_delete_cell_id) self._dep_tracker.start_transaction() self._exec_unit_container.start_transaction() try: # 4. Test if the code is new or a redefinition if self._exec_unit_container.contains_display_id( code_obj.display_id): current_exec_unit = self._update_existing_exec_unit( code_obj, cell_id) else: current_exec_unit = self._create_new_exec_unit( code_obj, cell_id) except Exception as e: # rollback changes made to dep graph self._dep_tracker.rollback() self._exec_unit_container.rollback() raise e else: self._dep_tracker.commit() self._exec_unit_container.commit() return current_exec_unit def _output_exec_results(self, exec_unit: ExecutionUnitInfo, request: RequestInfo, is_current: bool, exec_result: ExecutionResult): # 6b. Determine whether the current execution unit will be # directly display or update message_mode = 'update_display_data' if not is_current else 'display_data' # 6c. Create rich outputs for captured output if exec_result.output is not None: data, md = self.formatter.format(exec_result.output) else: data, md = {}, {} stdout, stderr = exec_result.stdout.getvalue( ), exec_result.stderr.getvalue() # 6d. For the captured output value, stdout, and stderr # send appropriate responses back to the front-end if len(stdout) > 0: self.session.send(self.iopub_socket, message_mode, content={ 'data': { 'text/plain': stdout }, 'metadata': {}, 'transient': { 'display_id': exec_unit.stdout_display_id } }, parent=request.parent) if exec_result.output is not None: self.session.send(self.iopub_socket, message_mode, content={ 'data': data, 'metadata': md, 'transient': { 'display_id': exec_unit.display_id } }, parent=request.parent) if len(stderr) > 0: self.session.send(self.iopub_socket, 'stream', content={ 'name': 'stderr', 'text': stderr }, parent=request.parent) def _log(self, request, text): self.session.send(self.iopub_socket, 'stream', content={ 'name': 'stdout', 'text': text + '\n' }, parent=request.parent) async def do_execute(self, request: RequestInfo): try: # 1. Create code object code_obj = CodeObject(request.code, self._key, self._execution_ctx.ns_manager) # 2. Extract metadata (both are optional) cell_id = request.metadata[ 'cellId'] if 'cellId' in request.metadata else None deleted_cell_ids = request.metadata[ 'deletedCells'] if 'deletedCells' in request.metadata else None # Doesn't need lock because the code inside _update_kernel_state # will never await. If that ever changes, uncomment lock below async with self._inner_state_lock: current_exec_unit = self._update_kernel_state( code_obj, cell_id, deleted_cell_ids) # 5. Compute the dependant execution units which must be rerun, and # add the current execution unit to the front of the list descendants = self._dep_tracker.get_descendants( code_obj.display_id) descendant_exec_units = list( map( lambda display_id: self._exec_unit_container. get_by_display_id(display_id), descendants)) block = ExecBlock(request, current_exec_unit, descendant_exec_units) await self._execution_queue.put(block) except Exception: etype, value, tb = sys.exc_info() stb = self.KernelTB.structured_traceback(etype, value, tb) formatted_lines = self.KernelTB.stb2text(stb).splitlines() error_content = { 'ename': str(etype), 'evalue': str(value), 'traceback': formatted_lines } if not request.silent: self.session.send(self.iopub_socket, 'error', content=error_content, parent=request.parent) error_content['status'] = 'error' return error_content return { 'status': 'ok', 'execution_count': request.execution_count, }
class ReactivePythonKernel(Kernel): implementation = 'reactivepy' implementation_version = __version__ language_info = { 'name': 'python', 'version': sys.version.split()[0], 'nbconvert_exporter': 'python', 'mimetype': 'text/x-python', 'file_extension': '.py' } banner = '' # measured in seconds. This value is intended to delay the same amount as # 60fps does between frames REGULAR_GENERATOR_DELAY = 0.016666 def __init__(self, **kwargs): super().__init__(**kwargs) self._key = generate_id(size=32).encode('utf-8') self._eventloop = IOLoop.current().asyncio_loop self._inner_state_lock = Lock(loop=self._eventloop) self._dep_tracker = DependencyTracker() self._exec_unit_container = ExecUnitContainer() self.formatter = DisplayFormatter() self.ns_manager = BuiltInManager() self.initialize_builtins() self._execution_ctx = Executor(self._exec_unit_container, ns_manager=self.ns_manager) self.KernelTB = ultratb.AutoFormattedTB(mode='Plain', color_scheme='LightBG', tb_offset=1, debugger_cls=None) # mapping from variable name (target id) to (request id, generator # object) self._registered_generators = dict() def initialize_builtins(self): self.ns_manager.add_builtin('var_dependency_graph', self._var_dependency_graph) self.ns_manager.add_builtin('cell_dependency_graph', self._cell_dependency_graph) def _var_dependency_graph(self): h = Digraph() for start_node in self._dep_tracker.get_nodes(): if "-" in start_node: for dest_node in self._dep_tracker.get_neighbors(start_node): if "-" in dest_node: first = start_node[:start_node.find('-')] second = dest_node[:dest_node.find('-')] for char in "[]": first = first.replace(char, "") second = second.replace(char, "") h.edge(first, second) return h def _cell_dependency_graph(self): h = Digraph() for start_node in self._dep_tracker.get_nodes(): if "-" in start_node: for dest_node in self._dep_tracker.get_neighbors(start_node): if "-" in dest_node: first = self._exec_unit_container.get_by_display_id( start_node).pinning_cell second = self._exec_unit_container.get_by_display_id( dest_node).pinning_cell h.edge(first, second) return h def _update_existing_exec_unit(self, code_obj, cell_id): # 1. If it is a redefinition, get the old execution unit current_exec_unit = self._exec_unit_container.get_by_display_id( code_obj.display_id) # 2. If the cell is owned and the owning cell is not the cell that sent the execute request, raise an exception if current_exec_unit.is_pinned and current_exec_unit.pinning_cell != cell_id: raise RedefiningOwnedCellException old_code_obj = current_exec_unit.code_obj new_input_vars = set(code_obj.input_vars) old_input_vars = set(old_code_obj.input_vars) # 3. Compute the sets of edges to be added and deleted if the # variable dependencies of the cell have changed to_delete = old_input_vars - new_input_vars to_add = new_input_vars - old_input_vars # 4. Actually replace old code object, delete old edges, # add new edges exec_unit = self._exec_unit_container.get_by_display_id( code_obj.display_id) exec_unit.code_obj = code_obj for sym in to_delete: code_object_id = self._exec_unit_container.get_by_symbol( sym).code_obj.display_id self._dep_tracker.delete_edge(code_object_id, code_obj.display_id) for sym in to_add: code_object_id = self._exec_unit_container.get_by_symbol( sym).code_obj.display_id self._dep_tracker.add_edge(code_object_id, code_obj.display_id) # 5. Get the updated execution unit and return it return self._exec_unit_container.get_by_display_id(code_obj.display_id) def _create_new_exec_unit(self, code_obj, cell_id): # 1. Create new execution unit to hold code object + display # id + cell id data current_exec_unit = self._exec_unit_container.register( ExecutionUnitInfo(code_obj, pinning_cell=cell_id)) # 2. Add new node self._dep_tracker.add_node(code_obj.display_id) # 3. For each variable it depends on, find the complete set of # defining variables (all the variables that were defined in # the same code block), and create a dependency to them for sym in code_obj.input_vars: dep = self._exec_unit_container.get_by_symbol(sym) if dep is not None: code_object_id = dep.code_obj.display_id self._dep_tracker.add_edge(code_object_id, code_obj.display_id) else: raise DefinitionNotFoundException(str(sym)) return current_exec_unit async def _inner_execute_request_callback(self, request: RequestInfo): # Re-broadcast our input for the benefit of listening clients, and # start computing output if not request.silent: request.execution_count = self.execution_count = 1 + self.execution_count self._publish_execute_input(request.code, request.parent, request.execution_count) stop_on_error = request.content.get('stop_on_error', True) response_meta = self.init_metadata(request.parent) request.response_meta = response_meta request.stop_on_error = stop_on_error await self.do_execute(request) def _complete_execute_request(self, request: RequestInfo, reply_content): # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) request.response_meta = self.finish_metadata(request.parent, request.response_meta, reply_content) reply_msg = self.session.send(request.stream, u'execute_reply', reply_content, request.parent, metadata=request.response_meta, ident=request.ident) if not request.silent and reply_msg['content'][ 'status'] == u'error' and request.stop_on_error: self._abort_queues() def execute_request(self, stream, ident, parent): """handle an execute_request""" try: request = RequestInfo(stream, parent, ident) except BaseException: self.log.error("Got bad msg: ") self.log.error("%s", parent) return self._eventloop.create_task( self._inner_execute_request_callback(request)) def _update_kernel_state(self, code_obj, cell_id, deleted_cell_ids): # 1. If deletedCells was passed, then unpin all execution units # that were previously attached to a cell if deleted_cell_ids is not None: for to_delete_cell_id in deleted_cell_ids: self._exec_unit_container.unpin_exec_unit(to_delete_cell_id) self._dep_tracker.start_transaction() self._exec_unit_container.start_transaction() try: # 2. Test if the code is new or a redefinition if self._exec_unit_container.contains_display_id( code_obj.display_id): current_exec_unit = self._update_existing_exec_unit( code_obj, cell_id) else: current_exec_unit = self._create_new_exec_unit( code_obj, cell_id) except Exception as e: # 3a. rollback changes made to dep graph self._dep_tracker.rollback() self._exec_unit_container.rollback() raise e else: # 3b. commit changes made to dep graph self._dep_tracker.commit() self._exec_unit_container.commit() return current_exec_unit def _output_exec_results(self, exec_unit: ExecutionUnitInfo, request: RequestInfo, is_current: bool, exec_result: ExecutionResult): # 6b. Determine whether the current execution unit will be # directly display or update message_mode = 'update_display_data' if not is_current else 'display_data' # 6c. Create rich outputs for captured output if exec_result.output is not None: data, md = self.formatter.format(exec_result.output) else: data, md = {}, {} stdout, stderr = exec_result.stdout.getvalue( ), exec_result.stderr.getvalue() # 6d. For the captured output value, stdout, and stderr # send appropriate responses back to the front-end if len(stdout) > 0: self.session.send(self.iopub_socket, message_mode, content={ 'data': { 'text/plain': stdout }, 'metadata': {}, 'transient': { 'display_id': exec_unit.stdout_display_id } }, parent=request.parent) if exec_result.output is not None: self.session.send(self.iopub_socket, message_mode, content={ 'data': data, 'metadata': md, 'transient': { 'display_id': exec_unit.display_id } }, parent=request.parent) if len(stderr) > 0: self.session.send(self.iopub_socket, 'stream', content={ 'name': 'stderr', 'text': stderr }, parent=request.parent) async def do_execute(self, request: RequestInfo): try: # 1. Create code object code_obj = CodeObject(request.code, self._key, self._execution_ctx.ns_manager) # 2. Extract metadata (both are optional) cell_id = request.metadata[ 'cellId'] if 'cellId' in request.metadata else None deleted_cell_ids = request.metadata[ 'deletedCells'] if 'deletedCells' in request.metadata else None # 3. Update internal graph current_exec_unit = self._update_kernel_state( code_obj, cell_id, deleted_cell_ids) # 4. Compute the dependant execution units which must be rerun, and # add the current execution unit to the front of the list descendants = list( map(self._exec_unit_container.get_by_display_id, self._dep_tracker.get_descendants(code_obj.display_id))) is_awaitable, is_gen, is_async_gen = False, False, False current_result = self._execution_ctx.run_cell( current_exec_unit.code_obj.code, current_exec_unit.display_id) if current_result.output is not None: is_awaitable, is_gen, is_async_gen = inspect_output_attrs( current_result.output) # If the output is awaitable, wait for it and then replace # the old output if is_awaitable: current_result.output = await current_result.output # If the output is a regular generator, wrap it in an async # generator that will add a very small delay if is_gen: current_result.output = convert_gen_to_async( current_result.output, ReactivePythonKernel.REGULAR_GENERATOR_DELAY)() is_async_gen = True is_gen = False if is_async_gen: async_gen_obj = current_result.output current_result.output = await anext( current_result.output, None) if is_awaitable or is_gen or is_async_gen: self._execution_ctx.update_ns( {current_result.target_id: current_result.output}) if not request.silent: self._output_exec_results(current_exec_unit, request, True, current_result) self._complete_execute_request( request, { 'status': 'ok', 'execution_count': request.execution_count }) # If the result output was not an async generator, # then execute descendants in this "thread" of control. # The `_start_new_async_iter` will handle running descendants and output their stuff. if not is_async_gen: # Return control to event loop before executing descendants await asyncio.sleep(0) for exec_unit in descendants: await self._run_descendant(exec_unit, request) # If it is an async generator, either replace the old # generator that was active for this target name or # register this completely new generator. Remove the old # generator by canceling it if is_async_gen: await self._start_new_async_iter(current_exec_unit, async_gen_obj, current_result.target_id, request) except Exception: etype, value, tb = sys.exc_info() stb = self.KernelTB.structured_traceback(etype, value, tb) formatted_lines = self.KernelTB.stb2text(stb).splitlines() error_content = { 'ename': str(etype), 'evalue': str(value), 'traceback': formatted_lines } if not request.silent: self.session.send(self.iopub_socket, 'error', content=error_content, parent=request.parent) error_content['status'] = 'error' self._complete_execute_request(request, error_content) return async def _run_descendant(self, exec_unit: ExecutionUnitInfo, request: RequestInfo): """ Run descendant of an iterator or execute request. Also output new values to front end. """ result = self._execution_ctx.run_cell(exec_unit.code_obj.code, exec_unit.display_id) # Set vars ahead to provide defaults is_async_gen = False async_gen_obj = None if result.output is not None: is_awaitable, is_gen, is_async_gen = inspect_output_attrs( result.output) # If the output is awaitable, wait for it and then replace # the old output if is_awaitable: result.output = await result.output # If the output is a regular generator, wrap it in an async # generator that will add a very small delay if is_gen: result.output = convert_gen_to_async( result.output, ReactivePythonKernel.REGULAR_GENERATOR_DELAY)() is_async_gen = True is_gen = False # If it is an async generator, either replace the old # generator that was active for this target name or # register this completely new generator. Remove the old # generator by canceling it if is_async_gen: async_gen_obj = result.output result.output = await anext(result.output, None) if is_awaitable or is_gen or is_async_gen: self._execution_ctx.update_ns( {result.target_id: result.output}) if not request.silent: self._output_exec_results(exec_unit, request, False, result) if is_async_gen: await self._start_new_async_iter(exec_unit, async_gen_obj, result.target_id, request) async def _start_new_async_iter(self, exec_unit: ExecutionUnitInfo, async_gen_obj, target_id: str, request: RequestInfo): """Attempt to start a new async generator running. This will check for previous generators for this same variable, and cancel them. """ # Check if a previous iterator exists for this variable. if exec_unit.display_id in self._registered_generators: old_start, old_task = self._registered_generators[ exec_unit.display_id] new_start = now() if new_start > old_start: old_task.cancel() new_task = self._eventloop.create_task( self._run_async_iter(new_start, exec_unit, async_gen_obj, target_id, request)) self._registered_generators[ exec_unit.display_id] = new_start, new_task else: new_start = now() new_task = self._eventloop.create_task( self._run_async_iter(new_start, exec_unit, async_gen_obj, target_id, request)) self._registered_generators[ exec_unit.display_id] = new_start, new_task async def _run_async_iter(self, started, exec_unit: ExecutionUnitInfo, async_gen_obj, target_id: str, request: RequestInfo): try: while True: # Get next value and update namespace try: exec_result = await self._execution_ctx.run_coroutine( anext(async_gen_obj), target_id, nohandle_exceptions=(StopAsyncIteration, GeneratorExit)) except StopAsyncIteration as e: # Generator is complete break except GeneratorExit as e: # Generator cancelled break # Output values to the front end self._output_exec_results(exec_unit, request, False, exec_result) descendant_exec_units = list( map( self._exec_unit_container.get_by_display_id, self._dep_tracker.get_descendants( exec_unit.display_id))) # For each descendant, run them for descendant in descendant_exec_units: await self._run_descendant(descendant, request) # Return control to eventloop in between steps of generator await asyncio.sleep(0) except Exception: etype, value, tb = sys.exc_info() stb = self.KernelTB.structured_traceback(etype, value, tb) formatted_lines = self.KernelTB.stb2text(stb).splitlines() error_content = { 'ename': str(etype), 'evalue': str(value), 'traceback': formatted_lines } if not request.silent: self.session.send(self.iopub_socket, 'error', content=error_content, parent=request.parent)
class PickyKernel(Kernel): """A kernel that accepts kernel magics which configure the environment""" implementation = "picky" implementation_version = __version__ # This banner only shows on `jupyter console` (at the command line). banner = """Pick, the kernel for choosy users! ⛏ Read more about it at https://github.com/nteract/pick """ # NOTE: This may not match the underlying kernel we launch. However, we need a default # for the initial launch. # TODO: Dynamically send over the underlying kernel's kernel_info_reply later language_info = { "name": "python", "version": sys.version.split()[0], "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": sys.version_info[0] }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py", } default_kernel = None default_config = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Ensure the kernel we work with uses Futures on recv, so we can await them self.context = ctx = Context.instance() # Our subscription to messages from the kernel we launch self.iosub = ctx.socket(zmq.SUB) self.iosub.subscribe = b"" # From kernelapp.py, shell_streams are typically shell_stream, control_stream self.shell_stream = self.shell_streams[0] # Start with no child kernel self.child_kernel = None self.kernel_config = None self.acquiring_kernel = asyncio.Lock() self.kernel_launched = asyncio.Event() self.display_formatter = DisplayFormatter() def start(self): """Start the PickyKernel and its event loop""" super().start() loop = IOLoop.current() # Collect and send all IOPub messages, for all time # TODO: Check errors from this loop and restart as needed (or shut down the kernel) loop.add_callback(self.relay_iopub_messages) async def relay_iopub_messages(self): """Relay messages received by the Picky Kernel to the consumer client (e.g. notebook) """ while True: msg = await self.iosub.recv_multipart() # Send the message up to the consumer (for example, the notebook) self.iopub_socket.send_multipart(msg) async def start_kernel(self, name=None, config=None): # Create a connection file that is named as a child of this kernel base, ext = os.path.splitext(self.parent.connection_file) connection_file = "{base}-child{ext}".format( base=base, ext=ext, ) subkernel = subkernels.get_subkernel(name) try: km = await subkernel.launch( config=config, session=self.session, context=self.context, connection_file=connection_file, ) except Exception as err: self.log.error(err) kernel = KernelProxy(manager=km, shell_upstream=self.shell_stream) self.iosub.connect(kernel.iopub_url) # Make sure the kernel is really started. We do that by using # kernel_info_requests here. # # In the future we should incorporate kernel logs (output of the kernel process), then # and send back all the information back to the user as display output. # Create a temporary KernelClient for waiting for the kernel to start kc = km.client() kc.start_channels() # Wait for kernel info reply on shell channel while True: self.log.debug("querying kernel info") kc.kernel_info(reply=False) try: msg = await kc.shell_channel.get_msg(timeout=1) except Empty: pass else: if msg["msg_type"] == "kernel_info_reply": # Now we know the kernel is (mostly) ready. # However, most kernels are not quite ready at this point to send # on more execution. # # Do we wait for a further status: idle on iopub? # Wait for idle? # Wait for particular logs from the stdout of the kernel process? break if not await kc.is_alive(): # TODO: Emit child kernel death message into the notebook output self.log.error("Kernel died while launching") raise RuntimeError( "Kernel died before replying to kernel_info") # Wait before sending another kernel info request await asyncio.sleep(0.1) # Flush IOPub channel on our (temporary) kernel client while True: try: msg = await kc.iopub_channel.get_msg(timeout=0.2) except Empty: break # Clean up our temporary kernel client kc.stop_channels() # Inform all waiters for the kernel that it is ready self.kernel_launched.set() return kernel async def get_kernel(self): """Get a launched child kernel""" # Ensure that the kernel is launched await self.kernel_launched.wait() if self.child_kernel is None: self.log.error("the child kernel was not available") return self.child_kernel async def queue_before_relay(self, stream, ident, parent): """Queue messages before sending between the child and parent kernels.""" if not self.kernel_launched.is_set(): self._publish_status("busy") kernel = await self.get_kernel() self._publish_status("idle") self.session.send(kernel.shell, parent, ident=ident) def display(self, obj, parent, display_id=False, update=False): """Publish a rich format of an object from our picky kernel, associated with the parent message""" data, metadata = self.display_formatter.format(obj) if metadata is None: metadata = {} transient = {} if display_id: transient = {"display_id": display_id} content = {"data": data, "metadata": metadata, "transient": transient} self.session.send( self.iopub_socket, "update_display_data" if update else "display_data", content, parent=parent, ident=self._topic("display_data"), ) def _publish_error(self, exc_info, parent=None): """send error on IOPub""" exc_type, exception, traceback = exc_info content = { "ename": exc_type.__name__, "evalue": str(exception), "traceback": format_tb(traceback), } self.session.send( self.iopub_socket, "error", content, parent=parent, ident=self._topic("error"), ) def _publish_execute_reply_error(self, exc_info, ident, parent): exc_type, exception, traceback = exc_info reply_content = { "status": "error", "ename": exc_type.__name__, "evalue": str(exception), "traceback": format_tb(traceback), # Since this isn't the underlying kernel "execution_count": None, } self.session.send( self.shell_stream, "execute_reply", reply_content, parent=parent, metadata={"status": "error"}, ident=ident, ) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send( self.iopub_socket, "status", {"execution_state": status}, parent=parent or self._parent_header, ident=self._topic("status"), metadata={"picky": True}, ) async def async_execute_request(self, stream, ident, parent): """process an execution request, sending it on to the child kernel or launching one if it has not been started. """ # Announce that we are busy handling this request self._publish_status("busy") # Only one kernel can be acquired at the same time async with self.acquiring_kernel: # Check for configuration code first content = parent["content"] code = content["code"] config, kernel_name = self.parse_cell(content["code"]) has_config = bool(config) if has_config: # Report back that we'll be running the config code # NOTE: We are, for the time being, ignoring the silent flag, store_history, etc. self._publish_execute_input(code, parent, self.execution_count) # User is attempting to run a cell with config after the kernel is started, # so we inform them and bail if has_config and self.child_kernel is not None: self.display( Markdown(f"""## Kernel already configured and launched. You can only run the `%%kernel.{kernel_name}` cell at the top of your notebook and the start of your session. Please **restart your kernel** and run the cell again if you want to change configuration. """), parent=parent, ) # Complete the "execution request" so the jupyter client (e.g. the notebook) thinks # execution is finished reply_content = { "status": "error", # Since our result is not part of `In` or `Out`, ensure # that the execution count is unset "execution_count": None, "user_expressions": {}, "payload": {}, } metadata = { "parametrized-kernel": True, "status": reply_content["status"], } self.session.send( stream, "execute_reply", reply_content, parent, metadata=metadata, ident=ident, ) self._publish_status("idle") return kernel_display_id = hexlify(os.urandom(8)).decode("ascii") # Launching a custom kernel if self.child_kernel is None and has_config: # Start a kernel now # If there's config set, we launch with that config # If the user is requesting the kernel with config, launch it! self.display( Markdown("Launching customized runtime..."), display_id=kernel_display_id, parent=parent, ) try: self.child_kernel = await self.start_kernel( kernel_name, config) except PickRegistrationException as err: # Get access to the exception info prior to doing any potential awaiting exc_info = sys.exc_info() self.log.info(exc_info) separator = "\n" self.display( Markdown( f"""## There is no kernel magic named `{kernel_name}` These are the available kernels: {separator.join([f"* `{kernel}`" for kernel in subkernels.list_subkernels()])} """), display_id=kernel_display_id, update=True, parent=parent, ) self.log.error(err) self._publish_execute_reply_error(exc_info, ident=ident, parent=parent) self._publish_status("idle", parent=parent) return except Exception as err: self.log.error(err) exc_info = sys.exc_info() self._publish_error(exc_info, parent=parent) self._publish_execute_reply_error(exc_info, ident=ident, parent=parent) self._publish_status("idle", parent=parent) return self.display( Markdown("Runtime ready!"), display_id=kernel_display_id, parent=parent, update=True, ) # Complete the "execution request" so the jupyter client (e.g. the notebook) thinks # execution is finished reply_content = { "status": "ok", # Our kernel setup is always the zero-th execution (In[] starts at 1) "execution_count": 0, # Note: user_expressions are not supported on our kernel creation magic "user_expressions": {}, "payload": {}, } metadata = { "parametrized-kernel": True, "status": reply_content["status"], } self.session.send( stream, "execute_reply", reply_content, parent, metadata=metadata, ident=ident, ) # With that, we're all done launching the customized kernel and # pushing updates on the kernel to the user. return # Start the default kernel to run code if self.child_kernel is None: self.display( Markdown("Preparing default kernel..."), parent=parent, display_id=kernel_display_id, ) self.child_kernel = await self.start_kernel( self.default_kernel, self.default_config) self.display( # Wipe out the previous message. # NOTE: The Jupyter notebook frontend ignores the case of an empty output for # an update_display_data so we have to publish some empty content instead. Markdown(""), parent=parent, display_id=kernel_display_id, update=True, ) self.session.send(self.child_kernel.shell, parent, ident=ident) def parse_cell(self, cell): if not cell.startswith("%%kernel."): return None, None try: # Split off our config from the kernel magic name magic_name, raw_config = cell.split("\n", 1) kernel_name = magic_name.split("%%kernel.")[1] return raw_config, kernel_name except Exception as err: self.log.error(err) return None, None def _log_task_exceptions(self, task, *args, **kwargs): try: task_exception = task.exception() if task_exception: self.log.error(task_exception) self.log.error(task.get_stack(limit=5)) except asyncio.CancelledError as err: self.log.error(err) except Exception as err: self.log.error(err) def relay_execute_to_kernel(self, stream, ident, parent): # Shove this execution onto the task queue task = asyncio.create_task( self.async_execute_request(stream, ident, parent)) task.add_done_callback(partial(self._log_task_exceptions, task)) def relay_to_kernel(self, stream, ident, parent): # relay_to_kernel is synchronous, and we rely on an asynchronous start # so we create each kernel message as a task... task = asyncio.create_task( self.queue_before_relay(stream, ident, parent)) task.add_done_callback(partial(self._log_task_exceptions, task)) execute_request = relay_execute_to_kernel inspect_request = relay_to_kernel complete_request = relay_to_kernel
# Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. from __future__ import print_function import matplotlib from matplotlib.backends.backend_agg import new_figure_manager, FigureCanvasAgg # analysis: ignore from matplotlib._pylab_helpers import Gcf from ipykernel.pylab.config import InlineBackend from IPython.core.formatters import DisplayFormatter from IPython.core.pylabtools import select_figure_formats import IPython import IPython.display import IPython.core.display df = DisplayFormatter() # fake thing which select_figure_formats will work on class qshell(): def __init__(self, df=None): self.display_formatter = df qpubcallback = None qclearcallback = None qipythoncallback = None shell = qshell(df) select_figure_formats(shell, { 'png' }) #,'svg','jpeg','pdf','retina'}) /more means multiple output mime types?
def __init__(self, **k): # connect with kdb super().__init__(**k) self._formatter = DisplayFormatter()
class EchoKernel(Kernel): implementation = 'Echo' implementation_version = '1.0' language = 'no-op' language_version = '0.1' language_info = { 'name': 'Any text', 'mimetype': 'text/plain', 'file_extension': '.txt', } banner = "Echo kernel - as useful as a parrot" def __init__(self, **k): # connect with kdb super().__init__(**k) self._formatter = DisplayFormatter() def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): if not silent: magic_code, code = strip_magic(code) stream_content = { 'name': 'stdout', 'text': str(user_expressions) + '\n' } self.send_response(self.iopub_socket, 'stream', stream_content) stream_content = {'name': 'stdout', 'text': '===\n'} self.send_response(self.iopub_socket, 'stream', stream_content) stream_content = {'name': 'stdout', 'text': code} self.send_response(self.iopub_socket, 'stream', stream_content) o = pd.DataFrame(dict( key=list('abc'), val=[code] * 3, )) # stream_content = {'data': {'text/html': '<b>hihihi</b>'}, # 'metadata': {}} # self.send_response(self.iopub_socket, 'display_data', stream_content) data, metadata = self._formatter.format(o) stream_content = { 'data': data, 'metadata': metadata, # 'data': { # 'text/html': '<code>bibidi</code>', # }, # 'metadata': { # }, 'execution_count': self.execution_count, } self.send_response( self.iopub_socket, 'execute_result', stream_content, ) return { 'status': 'ok', # The base class increments the execution count 'execution_count': self.execution_count, 'payload': [], 'user_expressions': {}, }
def publish_to_display(obj): output, _ = DisplayFormatter().format(obj) publish_display_data(output)