class SanitizeHTML(Preprocessor): # Bleach config. attributes = Any( config=True, default_value=ALLOWED_ATTRIBUTES, help="Allowed HTML tag attributes", ) tags = List( Unicode(), config=True, default_value=ALLOWED_TAGS, help="List of HTML tags to allow", ) styles = List( Unicode(), config=True, default_value=ALLOWED_STYLES, help="Allowed CSS styles if <style> tag is whitelisted" ) strip = Bool( config=True, default_value=False, help="If True, remove unsafe markup entirely instead of escaping" ) strip_comments = Bool( config=True, default_value=True, help="If True, strip comments from escaped HTML", ) # Display data config. safe_output_keys = Set( config=True, default_value={ 'metadata', # Not a mimetype per-se, but expected and safe. 'text/plain', 'text/latex', 'application/json', 'image/png', 'image/jpeg', }, help="Cell output mimetypes to render without modification", ) sanitized_output_types = Set( config=True, default_value={ 'text/html', 'text/markdown', }, help="Cell output types to display after escaping with Bleach.", ) def preprocess_cell(self, cell, resources, cell_index): """ Sanitize potentially-dangerous contents of the cell. Cell Types: raw: Sanitize literal HTML markdown: Sanitize literal HTML code: Sanitize outputs that could result in code execution """ if cell.cell_type == 'raw': # Sanitize all raw cells anyway. # Only ones with the text/html mimetype should be emitted # but erring on the side of safety maybe. cell.source = self.sanitize_html_tags(cell.source) return cell, resources elif cell.cell_type == 'markdown': cell.source = self.sanitize_html_tags(cell.source) return cell, resources elif cell.cell_type == 'code': cell.outputs = self.sanitize_code_outputs(cell.outputs) return cell, resources def sanitize_code_outputs(self, outputs): """ Sanitize code cell outputs. Removes 'text/javascript' fields from display_data outputs, and runs `sanitize_html_tags` over 'text/html'. """ for output in outputs: # These are always ascii, so nothing to escape. if output['output_type'] in ('stream', 'error'): continue data = output.data to_remove = [] for key in data: if key in self.safe_output_keys: continue elif key in self.sanitized_output_types: self.log.info("Sanitizing %s" % key) data[key] = self.sanitize_html_tags(data[key]) else: # Mark key for removal. (Python doesn't allow deletion of # keys from a dict during iteration) to_remove.append(key) for key in to_remove: self.log.info("Removing %s" % key) del data[key] return outputs def sanitize_html_tags(self, html_str): """ Sanitize a string containing raw HTML tags. """ return clean( html_str, tags=self.tags, attributes=self.attributes, styles=self.styles, strip=self.strip, strip_comments=self.strip_comments, )
class _Selection(LabeledWidget, ValueWidget, CoreWidget): """Base class for Selection widgets ``options`` can be specified as a list of values, list of (label, value) tuples, or a dict of {label: value}. The labels are the strings that will be displayed in the UI, representing the actual Python choices, and should be unique. If labels are not specified, they are generated from the values. When programmatically setting the value, a reverse lookup is performed among the options to check that the value is valid. The reverse lookup uses the equality operator by default, but another predicate may be provided via the ``equals`` keyword argument. For example, when dealing with numpy arrays, one may set equals=np.array_equal. Only labels are synced (values are converted to/from labels), so the labels should be unique. """ value = Any(help="Selected value").tag(sync=True, to_json=_value_to_label, from_json=_label_to_value) options = Union( [List(), Dict()], help= """List of values, or (label, value) tuples, or a dict of {label: value} pairs that the user can select. The labels are the strings that will be displayed in the UI, representing the actual Python choices, and should be unique. If labels are not specified, they are generated from the values. The keys are also available as _options_labels. """) _options_dict = Dict(read_only=True) _options_labels = Tuple(read_only=True).tag(sync=True) _options_values = Tuple(read_only=True) _model_module = Unicode('jupyter-js-widgets').tag(sync=True) _view_module = Unicode('jupyter-js-widgets').tag(sync=True) disabled = Bool(help="Enable or disable user changes").tag(sync=True) def __init__(self, *args, **kwargs): self.equals = kwargs.pop('equals', lambda x, y: x == y) super(_Selection, self).__init__(*args, **kwargs) def _make_options(self, x): """Standardize the options list format. The returned list should be in the format [('label', value), ('label', value), ...]. The input can be * a Mapping of labels to values * an iterable of values (of which at least one is not a list or tuple of length 2) * an iterable with entries that are lists or tuples of the form ('label', value) """ # Return a list of key-value pairs where the keys are strings # If x is a dict, convert it to list format. if isinstance(x, Mapping): return [(unicode_type(k), v) for k, v in x.items()] # If any entry of x is not a list or tuple of length 2, convert # the entries to unicode for the labels. for y in x: if not (isinstance(y, (list, tuple)) and len(y) == 2): return [(unicode_type(i), i) for i in x] # x is already in the correct format: a list of 2-tuples. # The first element of each tuple should be unicode, this might # not yet be the case. return [(unicode_type(k), v) for k, v in x] @validate('options') def _validate_options(self, proposal): """Handles when the options tuple has been changed. Setting options with a dict implies setting option labels from the keys of the dict. """ new = proposal['value'] options = self._make_options(new) self.set_trait('_options_dict', dict(options)) self.set_trait('_options_labels', [i[0] for i in options]) self.set_trait('_options_values', [i[1] for i in options]) return new @observe('options') def _value_in_options(self, change): "Ensure the value is an option; if not, set to the first value" # ensure that the chosen value is still one of the options if len(self.options) == 0: self.value = None else: try: _value_to_label(self.value, self) except KeyError: self.value = self._options_values[0] @validate('value') def _validate_value(self, proposal): value = proposal['value'] if len(self.options) == 0: if value is None: return value else: raise TraitError('Invalid selection: empty options list') else: try: _value_to_label(value, self) return value except KeyError: raise TraitError('Invalid selection')
class Kernel(SingletonConfigurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) @observe('eventloop') def _update_eventloop(self, change): """schedule call to eventloop from IOLoop""" loop = ioloop.IOLoop.instance() loop.add_callback(self.enter_eventloop) session = Instance(Session, allow_none=True) profile_dir = Instance('IPython.core.profiledir.ProfileDir', allow_none=True) shell_streams = List() control_stream = Instance(ZMQStream, allow_none=True) iopub_socket = Any() iopub_thread = Any() stdin_socket = Any() log = Instance(logging.Logger, allow_none=True) # identities: int_id = Integer(-1) ident = Unicode() @default('ident') def _default_ident(self): return unicode_type(uuid.uuid4()) # This should be overridden by wrapper kernels that implement any real # language. language_info = {} # any links that should go in the help menu help_links = List() # Private interface _darwin_app_nap = Bool(True, help="""Whether to use appnope for compatiblity with OS X App Nap. Only affects OS X >= 10.9. """ ).tag(config=True) # track associations with current request _allow_stdin = Bool(False) _parent_header = Dict() _parent_ident = Any(b'') # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005).tag(config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05).tag(config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() # set of aborted msg_ids aborted = Set() # Track execution count here. For IPython, we override this to use the # execution count we store in the shell. execution_count = 0 msg_types = [ 'execute_request', 'complete_request', 'inspect_request', 'history_request', 'comm_info_request', 'kernel_info_request', 'connect_request', 'shutdown_request', 'is_complete_request', # deprecated: 'apply_request', ] # add deprecated ipyparallel control messages control_msg_types = msg_types + ['clear_request', 'abort_request'] def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Build dict of handlers for message types self.shell_handlers = {} for msg_type in self.msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) self.control_handlers = {} for msg_type in self.control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) def dispatch_control(self, msg): """dispatch control requests""" idents,msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.deserialize(msg, content=True, copy=False) except: self.log.error("Invalid Control Message", exc_info=True) return self.log.debug("Control received: %s", msg) # Set the parent message for side effects. self.set_parent(idents, msg) self._publish_status(u'busy') header = msg['header'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: try: handler(self.control_stream, idents, msg) except Exception: self.log.error("Exception in control handler:", exc_info=True) sys.stdout.flush() sys.stderr.flush() self._publish_status(u'idle') def should_handle(self, stream, msg, idents): """Check whether a shell-channel message should be handled Allows subclasses to prevent handling of certain messages (e.g. aborted requests). """ msg_id = msg['header']['msg_id'] if msg_id in self.aborted: msg_type = msg['header']['msg_type'] # is it safe to assume a msg_id will not be resubmitted? self.aborted.remove(msg_id) reply_type = msg_type.split('_')[0] + '_reply' status = {'status' : 'aborted'} md = {'engine' : self.ident} md.update(status) self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) return False return True def dispatch_shell(self, stream, msg): """dispatch shell requests""" # flush control requests first if self.control_stream: self.control_stream.flush() idents,msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.deserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return # Set the parent message for side effects. self.set_parent(idents, msg) self._publish_status(u'busy') header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) if not self.should_handle(stream, msg, idents): return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: self.log.debug("%s: %s", msg_type, msg) self.pre_handler_hook() try: handler(stream, idents, msg) except Exception: self.log.error("Exception in message handler:", exc_info=True) finally: self.post_handler_hook() sys.stdout.flush() sys.stderr.flush() self._publish_status(u'idle') def pre_handler_hook(self): """Hook to execute before calling message handler""" # ensure default_int_handler during handler call self.saved_sigint_handler = signal(SIGINT, default_int_handler) def post_handler_hook(self): """Hook to execute after calling message handler""" signal(SIGINT, self.saved_sigint_handler) def enter_eventloop(self): """enter eventloop""" self.log.info("entering eventloop %s", self.eventloop) for stream in self.shell_streams: # flush any pending replies, # which may be skipped by entering the eventloop stream.flush(zmq.POLLOUT) # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel self.log.error("KeyboardInterrupt caught in kernel") continue else: # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break self.log.info("exiting eventloop") def start(self): """register dispatchers for streams""" if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_shell(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) # publish idle status self._publish_status('starting') def do_one_iteration(self): """step eventloop just once""" if self.control_stream: self.control_stream.flush() for stream in self.shell_streams: # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _publish_execute_input(self, code, parent, execution_count): """Publish the code request on the iopub stream.""" self.session.send(self.iopub_socket, u'execute_input', {u'code':code, u'execution_count': execution_count}, parent=parent, ident=self._topic('execute_input') ) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send(self.iopub_socket, u'status', {u'execution_state': status}, parent=parent or self._parent_header, ident=self._topic('status'), ) def set_parent(self, ident, parent): """Set the current parent_header Side effects (IOPub messages) and replies are associated with the request that caused them via the parent_header. The parent identity is used to route input_request messages on the stdin channel. """ self._parent_ident = ident self._parent_header = parent def send_response(self, stream, msg_or_type, content=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Send a response to the message we're currently processing. This accepts all the parameters of :meth:`jupyter_client.session.Session.send` except ``parent``. This relies on :meth:`set_parent` having been called for the current message. """ return self.session.send(stream, msg_or_type, content, self._parent_header, ident, buffers, track, header, metadata) def init_metadata(self, parent): """Initialize metadata. Run at the beginning of execution requests. """ return { 'started': datetime.now(), } def finish_metadata(self, parent, metadata, reply_content): """Finish populating metadata. Run after completing an execution request. """ return metadata def execute_request(self, stream, ident, parent): """handle an execute_request""" try: content = parent[u'content'] code = py3compat.cast_unicode_py2(content[u'code']) silent = content[u'silent'] store_history = content.get(u'store_history', not silent) user_expressions = content.get('user_expressions', {}) allow_stdin = content.get('allow_stdin', False) except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return stop_on_error = content.get('stop_on_error', True) metadata = self.init_metadata(parent) # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self.execution_count += 1 self._publish_execute_input(code, parent, self.execution_count) reply_content = self.do_execute(code, silent, store_history, user_expressions, allow_stdin) # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) metadata = self.finish_metadata(parent, metadata, reply_content) reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=metadata, ident=ident) self.log.debug("%s", reply_msg) if not silent and reply_msg['content']['status'] == u'error' and stop_on_error: self._abort_queues() def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): """Execute user code. Must be overridden by subclasses. """ raise NotImplementedError def complete_request(self, stream, ident, parent): content = parent['content'] code = content['code'] cursor_pos = content['cursor_pos'] matches = self.do_complete(code, cursor_pos) matches = json_clean(matches) completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) def do_complete(self, code, cursor_pos): """Override in subclasses to find completions. """ return {'matches' : [], 'cursor_end' : cursor_pos, 'cursor_start' : cursor_pos, 'metadata' : {}, 'status' : 'ok'} def inspect_request(self, stream, ident, parent): content = parent['content'] reply_content = self.do_inspect(content['code'], content['cursor_pos'], content.get('detail_level', 0)) # Before we send this object over, we scrub it for JSON usage reply_content = json_clean(reply_content) msg = self.session.send(stream, 'inspect_reply', reply_content, parent, ident) self.log.debug("%s", msg) def do_inspect(self, code, cursor_pos, detail_level=0): """Override in subclasses to allow introspection. """ return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False} def history_request(self, stream, ident, parent): content = parent['content'] reply_content = self.do_history(**content) reply_content = json_clean(reply_content) msg = self.session.send(stream, 'history_reply', reply_content, parent, ident) self.log.debug("%s", msg) def do_history(self, hist_access_type, output, raw, session=None, start=None, stop=None, n=None, pattern=None, unique=False): """Override in subclasses to access history. """ return {'status': 'ok', 'history': []} def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} content['status'] = 'ok' msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) @property def kernel_info(self): return { 'protocol_version': kernel_protocol_version, 'implementation': self.implementation, 'implementation_version': self.implementation_version, 'language_info': self.language_info, 'banner': self.banner, 'help_links': self.help_links, } def kernel_info_request(self, stream, ident, parent): content = {'status': 'ok'} content.update(self.kernel_info) msg = self.session.send(stream, 'kernel_info_reply', content, parent, ident) self.log.debug("%s", msg) def comm_info_request(self, stream, ident, parent): content = parent['content'] target_name = content.get('target_name', None) # Should this be moved to ipkernel? if hasattr(self, 'comm_manager'): comms = { k: dict(target_name=v.target_name) for (k, v) in self.comm_manager.comms.items() if v.target_name == target_name or target_name is None } else: comms = {} reply_content = dict(comms=comms, status='ok') msg = self.session.send(stream, 'comm_info_reply', reply_content, parent, ident) self.log.debug("%s", msg) def shutdown_request(self, stream, ident, parent): content = self.do_shutdown(parent['content']['restart']) self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', content, parent ) self._at_shutdown() # call sys.exit after a short delay loop = ioloop.IOLoop.instance() loop.add_timeout(time.time()+0.1, loop.stop) def do_shutdown(self, restart): """Override in subclasses to do things when the frontend shuts down the kernel. """ return {'status': 'ok', 'restart': restart} def is_complete_request(self, stream, ident, parent): content = parent['content'] code = content['code'] reply_content = self.do_is_complete(code) reply_content = json_clean(reply_content) reply_msg = self.session.send(stream, 'is_complete_reply', reply_content, parent, ident) self.log.debug("%s", reply_msg) def do_is_complete(self, code): """Override in subclasses to find completions. """ return {'status' : 'unknown', } #--------------------------------------------------------------------------- # Engine methods (DEPRECATED) #--------------------------------------------------------------------------- def apply_request(self, stream, ident, parent): self.log.warn("""apply_request is deprecated in kernel_base, moving to ipyparallel.""") try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] except: self.log.error("Got bad msg: %s", parent, exc_info=True) return md = self.init_metadata(parent) reply_content, result_buf = self.do_apply(content, bufs, msg_id, md) # flush i/o sys.stdout.flush() sys.stderr.flush() md = self.finish_metadata(parent, md, reply_content) self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident,buffers=result_buf, metadata=md) def do_apply(self, content, bufs, msg_id, reply_metadata): """DEPRECATED""" raise NotImplementedError #--------------------------------------------------------------------------- # Control messages (DEPRECATED) #--------------------------------------------------------------------------- def abort_request(self, stream, ident, parent): """abort a specific msg by id""" self.log.warn("abort_request is deprecated in kernel_base. It os only part of IPython parallel") msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, string_types): msg_ids = [msg_ids] if not msg_ids: self._abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" self.log.warn("clear_request is deprecated in kernel_base. It os only part of IPython parallel") content = self.do_clear() self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content = content) def do_clear(self): """DEPRECATED""" raise NotImplementedError #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _topic(self, topic): """prefixed topic for IOPub messages""" base = "kernel.%s" % self.ident return py3compat.cast_bytes("%s.%s" % (base, topic)) def _abort_queues(self): for stream in self.shell_streams: if stream: self._abort_queue(stream) def _abort_queue(self, stream): poller = zmq.Poller() poller.register(stream.socket, zmq.POLLIN) while True: idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' status = {'status' : 'aborted'} md = {'engine' : self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. poller.poll(50) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this " "frontend does not support stdin.") def getpass(self, prompt=''): """Forward getpass to frontends Raises ------ StdinNotImplentedError if active frontend doesn't support stdin. """ if not self._allow_stdin: raise StdinNotImplementedError( "getpass was called, but this frontend does not support input requests." ) return self._input_request(prompt, self._parent_ident, self._parent_header, password=True, ) def raw_input(self, prompt=''): """Forward raw_input to frontends Raises ------ StdinNotImplentedError if active frontend doesn't support stdin. """ if not self._allow_stdin: raise StdinNotImplementedError( "raw_input was called, but this frontend does not support input requests." ) return self._input_request(str(prompt), self._parent_ident, self._parent_header, password=False, ) def _input_request(self, prompt, ident, parent, password=False): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # flush the stdin socket, to purge stale replies while True: try: self.stdin_socket.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: break else: raise # Send the input request. content = json_clean(dict(prompt=prompt, password=password)) self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) except KeyboardInterrupt: # re-raise KeyboardInterrupt, to truncate traceback raise KeyboardInterrupt else: break try: value = py3compat.unicode_to_str(reply['content']['value']) except: self.log.error("Bad input_reply: %s", parent) value = '' if value == '\x04': # EOF raise EOFError return value def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name = 'ipython-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('ipykernel.ipkernel.IPythonKernel', klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the IPKernelApp entry point to configure and launch kernels other than IPython's own. """).tag(config=True) kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) context = Any() shell_socket = Any() control_socket = Any() debugpy_socket = Any() debug_shell_socket = Any() stdin_socket = Any() iopub_socket = Any() iopub_thread = Any() control_thread = Any() _ports = Dict() subcommands = { 'install': ('ipykernel.kernelspec.InstallIPythonKernelSpecApp', 'Install the IPython kernel'), } # connection info: connection_dir = Unicode() @default('connection_dir') def _default_connection_dir(self): return jupyter_runtime_dir() @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool( False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool( False, help="redirect stderr to the null device").tag(config=True) trio_loop = Bool(False, help="Set main event loop.").tag(config=True) quiet = Bool( True, help="Only send stdout/stderr to output stream").tag(config=True) outstream_class = DottedObjectName( 'ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) displayhook_class = DottedObjectName( 'ipykernel.displayhook.ZMQDisplayHook', help="The importstring for the DisplayHook factory").tag(config=True) capture_fd_output = Bool( True, help= """Attempt to capture and forward low-level output, e.g. produced by Extension libraries. """, ).tag(config=True) # polling parent_handle = Integer( int(os.environ.get('JPY_PARENT_PID') or 0), help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """).tag(config=True) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """).tag(config=True) def init_crash_handler(self): sys.excepthook = self.excepthook def excepthook(self, etype, evalue, tb): # write uncaught traceback to 'real' stderr, not zmq-forwarder traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle and self.parent_handle != 1: # PID 1 (init) is special and will never go away, # only be reassigned. # Parent polling doesn't work if ppid == 1 to start with. self.poller = ParentPollerUnix() def _try_bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def _bind_socket(self, s, port): try: win_in_use = errno.WSAEADDRINUSE except AttributeError: win_in_use = None # Try up to 100 times to bind a port when in conflict to avoid # infinite attempts in bad setups max_attempts = 1 if port else 100 for attempt in range(max_attempts): try: return self._try_bind_socket(s, port) except zmq.ZMQError as ze: # Raise if we have any error not related to socket binding if ze.errno != errno.EADDRINUSE and ze.errno != win_in_use: raise if attempt == max_attempts - 1: raise def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except OSError: pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except OSError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, and I'll create it in this directory: os.makedirs(os.path.dirname(self.abs_connection_file), mode=0o700, exist_ok=True) # Also, I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) assert self.context is None, "init_sockets cannot be called twice!" self.context = context = zmq.Context() atexit.register(self.close) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) if hasattr(zmq, 'ROUTER_HANDOVER'): # set router-handover to workaround zeromq reconnect problems # in certain rare circumstances # see ipython/ipykernel#270 and zeromq/libzmq#2892 self.shell_socket.router_handover = \ self.stdin_socket.router_handover = 1 self.init_control(context) self.init_iopub(context) def init_control(self, context): self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) self.debugpy_socket = context.socket(zmq.STREAM) self.debugpy_socket.linger = 1000 self.debug_shell_socket = context.socket(zmq.DEALER) self.debug_shell_socket.linger = 1000 if self.shell_socket.getsockopt(zmq.LAST_ENDPOINT): self.debug_shell_socket.connect( self.shell_socket.getsockopt(zmq.LAST_ENDPOINT)) if hasattr(zmq, 'ROUTER_HANDOVER'): # set router-handover to workaround zeromq reconnect problems # in certain rare circumstances # see ipython/ipykernel#270 and zeromq/libzmq#2892 self.control_socket.router_handover = 1 self.control_thread = ControlThread(daemon=True) def init_iopub(self, context): self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.configure_tornado_logger() self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) self.iopub_thread.start() # backward-compat: wrap iopub socket API in background thread self.iopub_socket = self.iopub_thread.background_socket def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def close(self): """Close zmq sockets in an orderly fashion""" # un-capture IO before we start closing channels self.reset_io() self.log.info("Cleaning up sockets") if self.heartbeat: self.log.debug("Closing heartbeat channel") self.heartbeat.context.term() if self.iopub_thread: self.log.debug("Closing iopub channel") self.iopub_thread.stop() self.iopub_thread.close() if self.control_thread and self.control_thread.is_alive(): self.log.debug("Closing control thread") self.control_thread.stop() self.control_thread.join() if self.debugpy_socket and not self.debugpy_socket.closed: self.debugpy_socket.close() if self.debug_shell_socket and not self.debug_shell_socket.closed: self.debug_shell_socket.close() for channel in ('shell', 'control', 'stdin'): self.log.debug("Closing %s channel", channel) socket = getattr(self, channel + "_socket", None) if socket and not socket.closed: socket.close() self.log.debug("Terminating zmq context") self.context.term() self.log.debug("Terminated zmq context") def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) # unless log-level is CRITICAL (--quiet) if not self.parent_handle and self.log_level < logging.CRITICAL: print(_ctrl_c_message, file=sys.__stdout__) for line in lines: print(line, file=sys.__stdout__) self._ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) if sys.stdout is not None: sys.stdout.flush() e_stdout = None if self.quiet else sys.__stdout__ e_stderr = None if self.quiet else sys.__stderr__ if not self.capture_fd_output: outstream_factory = partial(outstream_factory, watchfd=False) sys.stdout = outstream_factory(self.session, self.iopub_thread, 'stdout', echo=e_stdout) if sys.stderr is not None: sys.stderr.flush() sys.stderr = outstream_factory(self.session, self.iopub_thread, "stderr", echo=e_stderr) if hasattr(sys.stderr, "_original_stdstream_copy"): for handler in self.log.handlers: if isinstance(handler, StreamHandler) and ( handler.stream.buffer.fileno() == 2): self.log.debug( "Seeing logger to stderr, rerouting to raw filedescriptor." ) handler.stream = TextIOWrapper( FileIO(sys.stderr._original_stdstream_copy, "w")) if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) self.displayhook = displayhook_factory(self.session, self.iopub_socket) sys.displayhook = self.displayhook self.patch_io() def reset_io(self): """restore original io restores state after init_io """ sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ sys.displayhook = sys.__displayhook__ def patch_io(self): """Patch important libraries that can't handle sys.stdout forwarding""" try: import faulthandler except ImportError: pass else: # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable # change default file to __stderr__ from forwarded stderr faulthandler_enable = faulthandler.enable def enable(file=sys.__stderr__, all_threads=True, **kwargs): return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) faulthandler.enable = enable if hasattr(faulthandler, 'register'): faulthandler_register = faulthandler.register def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): return faulthandler_register(signum, file=file, all_threads=all_threads, chain=chain, **kwargs) faulthandler.register = register def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket, self.control_thread.io_loop) debugpy_stream = ZMQStream(self.debugpy_socket, self.control_thread.io_loop) self.control_thread.start() kernel_factory = self.kernel_class.instance kernel = kernel_factory( parent=self, session=self.session, control_stream=control_stream, debugpy_stream=debugpy_stream, debug_shell_socket=self.debug_shell_socket, shell_stream=shell_stream, control_thread=self.control_thread, iopub_thread=self.iopub_thread, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports( {name + '_port': port for name, port in self._ports.items()}) self.kernel = kernel # Allow the displayhook to get the execution count self.displayhook.get_execution_count = lambda: kernel.execution_count def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Register inline backend as default # this is higher priority than matplotlibrc, # but lower priority than anything else (mpl.use() for instance). # This only affects matplotlib >= 1.5 if not os.environ.get('MPLBACKEND'): os.environ[ 'MPLBACKEND'] = 'module://matplotlib_inline.backend_inline' # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=sys.stderr) print(shell.InteractiveTB.stb2text(stb), file=sys.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def configure_tornado_logger(self): """ Configure the tornado logging.Logger. Must set up the tornado logger or else tornado will call basicConfig for the root logger which makes the root logger go to the real sys.stderr instead of the capture streams. This function mimics the setup of logging.basicConfig. """ logger = logging.getLogger('tornado') handler = logging.StreamHandler() formatter = logging.Formatter(logging.BASIC_FORMAT) handler.setFormatter(formatter) logger.addHandler(handler) def _init_asyncio_patch(self): """set default asyncio policy to be compatible with tornado Tornado 6 (at least) is not compatible with the default asyncio implementation on Windows Pick the older SelectorEventLoopPolicy on Windows if the known-incompatible default policy is in use. Support for Proactor via a background thread is available in tornado 6.1, but it is still preferable to run the Selector in the main thread instead of the background. do this as early as possible to make it a low priority and overrideable ref: https://github.com/tornadoweb/tornado/issues/2608 FIXME: if/when tornado supports the defaults in asyncio without threads, remove and bump tornado requirement for py38. Most likely, this will mean a new Python version where asyncio.ProactorEventLoop supports add_reader and friends. """ if sys.platform.startswith("win") and sys.version_info >= (3, 8): import asyncio try: from asyncio import ( WindowsProactorEventLoopPolicy, WindowsSelectorEventLoopPolicy, ) except ImportError: pass # not affected else: if type(asyncio.get_event_loop_policy() ) is WindowsProactorEventLoopPolicy: # WindowsProactorEventLoopPolicy is not compatible with tornado 6 # fallback to the pre-3.8 default of Selector asyncio.set_event_loop_policy( WindowsSelectorEventLoopPolicy()) def init_pdb(self): """Replace pdb with IPython's version that is interruptible. With the non-interruptible version, stopping pdb() locks up the kernel in a non-recoverable state. """ import pdb from IPython.core import debugger if hasattr(debugger, "InterruptiblePdb"): # Only available in newer IPython releases: debugger.Pdb = debugger.InterruptiblePdb pdb.Pdb = debugger.Pdb pdb.set_trace = debugger.set_trace @catch_config_error def initialize(self, argv=None): self._init_asyncio_patch() super().initialize(argv) if self.subapp is not None: return self.init_pdb() self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.write_connection_file() # Log connection info after writing connection file, so that the connection # file is definitely available at the time someone reads the log. self.log_connection_info() self.init_io() try: self.init_signal() except Exception: # Catch exception when initializing signal fails, eg when running the # kernel on a separate thread if self.log_level < logging.CRITICAL: self.log.error("Unable to initialize signal:", exc_info=True) self.init_kernel() # shell init steps self.init_path() self.init_shell() if self.shell: self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.subapp is not None: return self.subapp.start() if self.poller is not None: self.poller.start() self.kernel.start() self.io_loop = ioloop.IOLoop.current() if self.trio_loop: from ipykernel.trio_runner import TrioRunner tr = TrioRunner() tr.initialize(self.kernel, self.io_loop) try: tr.run() except KeyboardInterrupt: pass else: try: self.io_loop.start() except KeyboardInterrupt: pass
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp, ConnectionFileMixin): name = 'ipython-kernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session] # the kernel class, as an importstring kernel_class = Type('ipykernel.ipkernel.IPythonKernel', klass='ipykernel.kernelbase.Kernel', help="""The Kernel subclass to be used. This should allow easy re-use of the IPKernelApp entry point to configure and launch kernels other than IPython's own. """).tag(config=True) kernel = Any() poller = Any( ) # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat, allow_none=True) ports = Dict() subcommands = { 'install': ('ipykernel.kernelspec.InstallIPythonKernelSpecApp', 'Install the IPython kernel'), } # connection info: connection_dir = Unicode() @default('connection_dir') def _default_connection_dir(self): return jupyter_runtime_dir() @property def abs_connection_file(self): if os.path.basename(self.connection_file) == self.connection_file: return os.path.join(self.connection_dir, self.connection_file) else: return self.connection_file # streams, etc. no_stdout = Bool( False, help="redirect stdout to the null device").tag(config=True) no_stderr = Bool( False, help="redirect stderr to the null device").tag(config=True) quiet = Bool( True, help="Only send stdout/stderr to output stream").tag(config=True) outstream_class = DottedObjectName( 'ipykernel.iostream.OutStream', help="The importstring for the OutStream factory").tag(config=True) displayhook_class = DottedObjectName( 'ipykernel.displayhook.ZMQDisplayHook', help="The importstring for the DisplayHook factory").tag(config=True) # polling parent_handle = Integer( int(os.environ.get('JPY_PARENT_PID') or 0), help="""kill this process if its parent dies. On Windows, the argument specifies the HANDLE of the parent process, otherwise it is simply boolean. """).tag(config=True) interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), help="""ONLY USED ON WINDOWS Interrupt this process when the parent is signaled. """).tag(config=True) def init_crash_handler(self): sys.excepthook = self.excepthook def excepthook(self, etype, evalue, tb): # write uncaught traceback to 'real' stderr, not zmq-forwarder traceback.print_exception(etype, evalue, tb, file=sys.__stderr__) def init_poller(self): if sys.platform == 'win32': if self.interrupt or self.parent_handle: self.poller = ParentPollerWindows(self.interrupt, self.parent_handle) elif self.parent_handle and self.parent_handle != 1: # PID 1 (init) is special and will never go away, # only be reassigned. # Parent polling doesn't work if ppid == 1 to start with. self.poller = ParentPollerUnix() def _bind_socket(self, s, port): iface = '%s://%s' % (self.transport, self.ip) if self.transport == 'tcp': if port <= 0: port = s.bind_to_random_port(iface) else: s.bind("tcp://%s:%i" % (self.ip, port)) elif self.transport == 'ipc': if port <= 0: port = 1 path = "%s-%i" % (self.ip, port) while os.path.exists(path): port = port + 1 path = "%s-%i" % (self.ip, port) else: path = "%s-%i" % (self.ip, port) s.bind("ipc://%s" % path) return port def write_connection_file(self): """write connection info to JSON file""" cf = self.abs_connection_file self.log.debug("Writing connection file: %s", cf) write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port, control_port=self.control_port) def cleanup_connection_file(self): cf = self.abs_connection_file self.log.debug("Cleaning up connection file: %s", cf) try: os.remove(cf) except (IOError, OSError): pass self.cleanup_ipc_files() def init_connection_file(self): if not self.connection_file: self.connection_file = "kernel-%s.json" % os.getpid() try: self.connection_file = filefind(self.connection_file, ['.', self.connection_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) # This means I own it, and I'll create it in this directory: ensure_dir_exists(os.path.dirname(self.abs_connection_file), 0o700) # Also, I will clean it up: atexit.register(self.cleanup_connection_file) return try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_sockets(self): # Create a context, a session, and the kernel sockets. self.log.info("Starting the kernel at pid: %i", os.getpid()) context = zmq.Context.instance() # Uncomment this to try closing the context. # atexit.register(context.term) self.shell_socket = context.socket(zmq.ROUTER) self.shell_socket.linger = 1000 self.shell_port = self._bind_socket(self.shell_socket, self.shell_port) self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port) self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_socket.linger = 1000 self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port) self.control_socket = context.socket(zmq.ROUTER) self.control_socket.linger = 1000 self.control_port = self._bind_socket(self.control_socket, self.control_port) self.log.debug("control ROUTER Channel on port: %i" % self.control_port) if hasattr(zmq, 'ROUTER_HANDOVER'): # set router-handover to workaround zeromq reconnect problems # in certain rare circumstances # see ipython/ipykernel#270 and zeromq/libzmq#2892 self.shell_socket.router_handover = \ self.control_socket.router_handover = \ self.stdin_socket.router_handover = 1 self.init_iopub(context) def init_iopub(self, context): self.iopub_socket = context.socket(zmq.PUB) self.iopub_socket.linger = 1000 self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port) self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port) self.configure_tornado_logger() self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True) self.iopub_thread.start() # backward-compat: wrap iopub socket API in background thread self.iopub_socket = self.iopub_thread.background_socket def init_heartbeat(self): """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port) self.heartbeat.start() def log_connection_info(self): """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.connection_dir: # use shortname tail = basename else: tail = self.connection_file lines = [ "To connect another client to this kernel, use:", " --existing %s" % tail, ] # log connection info # info-level, so often not shown. # frontends should use the %connect_info magic # to see the connection info for line in lines: self.log.info(line) # also raw print to the terminal if no parent_handle (`ipython kernel`) # unless log-level is CRITICAL (--quiet) if not self.parent_handle and self.log_level < logging.CRITICAL: io.raw_print(_ctrl_c_message) for line in lines: io.raw_print(line) self.ports = dict(shell=self.shell_port, iopub=self.iopub_port, stdin=self.stdin_port, hb=self.hb_port, control=self.control_port) def init_blackhole(self): """redirects stdout/stderr to devnull if necessary""" if self.no_stdout or self.no_stderr: blackhole = open(os.devnull, 'w') if self.no_stdout: sys.stdout = sys.__stdout__ = blackhole if self.no_stderr: sys.stderr = sys.__stderr__ = blackhole def init_io(self): """Redirect input streams and set a display hook.""" if self.outstream_class: outstream_factory = import_item(str(self.outstream_class)) if sys.stdout is not None: sys.stdout.flush() e_stdout = None if self.quiet else sys.__stdout__ e_stderr = None if self.quiet else sys.__stderr__ sys.stdout = outstream_factory(self.session, self.iopub_thread, u'stdout', echo=e_stdout) if sys.stderr is not None: sys.stderr.flush() sys.stderr = outstream_factory(self.session, self.iopub_thread, u'stderr', echo=e_stderr) if self.displayhook_class: displayhook_factory = import_item(str(self.displayhook_class)) self.displayhook = displayhook_factory(self.session, self.iopub_socket) sys.displayhook = self.displayhook self.patch_io() def patch_io(self): """Patch important libraries that can't handle sys.stdout forwarding""" try: import faulthandler except ImportError: pass else: # Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible # updates to the upstream API and update accordingly (up-to-date as of Python 3.5): # https://docs.python.org/3/library/faulthandler.html#faulthandler.enable # change default file to __stderr__ from forwarded stderr faulthandler_enable = faulthandler.enable def enable(file=sys.__stderr__, all_threads=True, **kwargs): return faulthandler_enable(file=file, all_threads=all_threads, **kwargs) faulthandler.enable = enable if hasattr(faulthandler, 'register'): faulthandler_register = faulthandler.register def register(signum, file=sys.__stderr__, all_threads=True, chain=False, **kwargs): return faulthandler_register(signum, file=file, all_threads=all_threads, chain=chain, **kwargs) faulthandler.register = register def init_signal(self): signal.signal(signal.SIGINT, signal.SIG_IGN) def init_kernel(self): """Create the Kernel object itself""" shell_stream = ZMQStream(self.shell_socket) control_stream = ZMQStream(self.control_socket) kernel_factory = self.kernel_class.instance kernel = kernel_factory( parent=self, session=self.session, shell_streams=[shell_stream, control_stream], iopub_thread=self.iopub_thread, iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, user_ns=self.user_ns, ) kernel.record_ports( {name + '_port': port for name, port in self.ports.items()}) self.kernel = kernel # Allow the displayhook to get the execution count self.displayhook.get_execution_count = lambda: kernel.execution_count def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" # Register inline backend as default # this is higher priority than matplotlibrc, # but lower priority than anything else (mpl.use() for instance). # This only affects matplotlib >= 1.5 if not os.environ.get('MPLBACKEND'): os.environ[ 'MPLBACKEND'] = 'module://ipykernel.pylab.backend_inline' # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab` # to ensure that any exception is printed straight to stderr. # Normally _showtraceback associates the reply with an execution, # which means frontends will never draw it, as this exception # is not associated with any execute request. shell = self.shell _showtraceback = shell._showtraceback try: # replace error-sending traceback with stderr def print_tb(etype, evalue, stb): print("GUI event loop or pylab initialization failed", file=sys.stderr) print(shell.InteractiveTB.stb2text(stb), file=sys.stderr) shell._showtraceback = print_tb InteractiveShellApp.init_gui_pylab(self) finally: shell._showtraceback = _showtraceback def init_shell(self): self.shell = getattr(self.kernel, 'shell', None) if self.shell: self.shell.configurables.append(self) def init_extensions(self): super(IPKernelApp, self).init_extensions() # BEGIN HARDCODED WIDGETS HACK # Ensure ipywidgets extension is loaded if available extension_man = self.shell.extension_manager if 'ipywidgets' not in extension_man.loaded: try: extension_man.load_extension('ipywidgets') except ImportError as e: self.log.debug( 'ipywidgets package not installed. Widgets will not be available.' ) # END HARDCODED WIDGETS HACK def configure_tornado_logger(self): """ Configure the tornado logging.Logger. Must set up the tornado logger or else tornado will call basicConfig for the root logger which makes the root logger go to the real sys.stderr instead of the capture streams. This function mimics the setup of logging.basicConfig. """ logger = logging.getLogger('tornado') handler = logging.StreamHandler() formatter = logging.Formatter(logging.BASIC_FORMAT) handler.setFormatter(formatter) logger.addHandler(handler) @catch_config_error def initialize(self, argv=None): super(IPKernelApp, self).initialize(argv) if self.subapp is not None: return # register zmq IOLoop with tornado zmq_ioloop.install() self.init_blackhole() self.init_connection_file() self.init_poller() self.init_sockets() self.init_heartbeat() # writing/displaying connection info must be *after* init_sockets/heartbeat self.write_connection_file() # Log connection info after writing connection file, so that the connection # file is definitely available at the time someone reads the log. self.log_connection_info() self.init_io() self.init_signal() self.init_kernel() # shell init steps self.init_path() self.init_shell() if self.shell: self.init_gui_pylab() self.init_extensions() self.init_code() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request sys.stdout.flush() sys.stderr.flush() def start(self): if self.subapp is not None: return self.subapp.start() if self.poller is not None: self.poller.start() self.kernel.start() self.io_loop = ioloop.IOLoop.current() try: self.io_loop.start() except KeyboardInterrupt: pass
class QgridWidget(widgets.DOMWidget): """ The widget class which is instantiated by the 'show_grid' method, and can also be constructed directly. All of the parameters listed below can be read/updated after instantiation via attributes of the same name as the parameter (since they're implemented as traitlets). When new values are set for any of these options after instantiation (such as df, grid_options, etc), the change takes effect immediately by regenerating the SlickGrid control. Parameters ---------- df : DataFrame The DataFrame that will be displayed by this instance of QgridWidget. grid_options : dict Options to use when creating the SlickGrid control (i.e. the interactive grid). See the Notes section below for more information on the available options, as well as the default options that this widget uses. precision : integer The number of digits of precision to display for floating-point values. If unset, we use the value of `pandas.get_option('display.precision')`. show_toolbar : bool Whether to show a toolbar with options for adding/removing rows. Adding/removing rows is an experimental feature which only works with DataFrames that have an integer index. Notes ----- The following dictionary is used for ``grid_options`` if none are provided explicitly:: { 'fullWidthRows': True, 'syncColumnCellResize': True, 'forceFitColumns': True, 'defaultColumnWidth': 150, 'rowHeight': 28, 'enableColumnReorder': False, 'enableTextSelectionOnCells': True, 'editable': True, 'autoEdit': False, 'explicitInitialization': True, 'maxVisibleRows': 15, 'minVisibleRows': 8, 'sortable': True, 'filterable': True, 'highlightSelectedCell': False, 'highlightSelectedRow': True } Most of these options are SlickGrid options which are described in the `SlickGrid documentation <https://github.com/mleibman/SlickGrid/wiki/Grid-Options>`_. The exceptions are the last 6 options listed, which are options that were added specifically for Qgrid and therefore are not documented in the SlickGrid documentation. The first two, `maxVisibleRows` and `minVisibleRows`, allow you to set an upper and lower bound on the height of your Qgrid widget in terms of number of rows that are visible. The next two, `sortable` and `filterable`, control whether qgrid will allow the user to sort and filter, respectively. If you set `sortable` to False nothing will happen when the column headers are clicked. If you set `filterable` to False, the filter icons won't be shown for any columns. The last two, `highlightSelectedCell` and `highlightSelectedRow`, control how the styling of qgrid changes when a cell is selected. If you set `highlightSelectedCell` to True, the selected cell will be given a light blue border. If you set `highlightSelectedRow` to False, the light blue background that's shown by default for selected rows will be hidden. See Also -------- set_defaults : Permanently set global defaults for the parameters of the QgridWidget constructor, with the exception of the ``df`` parameter. set_grid_option : Permanently set global defaults for individual SlickGrid options. Does so by changing the default for the ``grid_options`` parameter of the QgridWidget constructor. Attributes ---------- df : DataFrame Get/set the DataFrame that's being displayed by the current instance. This DataFrame will NOT reflect any sorting/filtering/editing changes that are made via the UI. To get a copy of the DataFrame that does reflect sorting/filtering/editing changes, use the ``get_changed_df()`` method. grid_options : dict Get/set the SlickGrid options being used by the current instance. precision : integer Get/set the precision options being used by the current instance. show_toolbar : bool Get/set the show_toolbar option being used by the current instance. """ _view_name = Unicode('QgridView').tag(sync=True) _model_name = Unicode('QgridModel').tag(sync=True) _view_module = Unicode('qgrid').tag(sync=True) _model_module = Unicode('qgrid').tag(sync=True) _view_module_version = Unicode('1.0.2').tag(sync=True) _model_module_version = Unicode('1.0.2').tag(sync=True) _df = Instance(pd.DataFrame) _df_json = Unicode('', sync=True) _primary_key = List() _columns = Dict({}, sync=True) _filter_tables = Dict({}) _sorted_column_cache = Dict({}) _interval_columns = List([], sync=True) _period_columns = List([]) _string_columns = List([]) _sort_helper_columns = Dict({}) _initialized = Bool(False) _ignore_df_changed = Bool(False) _unfiltered_df = Instance(pd.DataFrame) _index_col_name = Unicode('qgrid_unfiltered_index', sync=True) _sort_col_suffix = Unicode('_qgrid_sort_column') _multi_index = Bool(False) _edited = Bool(False) _selected_rows = List([]) _viewport_range = Tuple(Integer(), Integer(), default_value=(0, 100)) _df_range = Tuple(Integer(), Integer(), default_value=(0, 100), sync=True) _row_count = Integer(0, sync=True) _sort_field = Any('', sync=True) _sort_ascending = Bool(True, sync=True) df = Instance(pd.DataFrame) precision = Integer(6, sync=True) grid_options = Dict(sync=True) show_toolbar = Bool(False, sync=True) def __init__(self, *args, **kwargs): self._initialized = False super(QgridWidget, self).__init__(*args, **kwargs) # register a callback for custom messages self.on_msg(self._handle_qgrid_msg) self._initialized = True if self.df is not None: self._update_df() def _grid_options_default(self): return defaults.grid_options def _precision_default(self): return defaults.precision def _show_toolbar_default(self): return defaults.show_toolbar def _update_df(self): self._ignore_df_changed = True # make a copy of the user's dataframe self._df = self.df.copy() # insert a column which we'll use later to map edits from # a filtered version of this df back to the unfiltered version self._df.insert(0, self._index_col_name, range(0, len(self._df))) # keep an unfiltered version to serve as the starting point # for filters, and the state we return to when filters are removed self._unfiltered_df = self._df.copy() self._update_table(update_columns=True, fire_data_change_event=False) self._ignore_df_changed = False def _rebuild_widget(self): self._update_df() self.send({'type': 'draw_table'}) def _df_changed(self): """Build the Data Table for the DataFrame.""" if self._ignore_df_changed or not self._initialized: return self._rebuild_widget() def _precision_changed(self): if not self._initialized: return self._rebuild_widget() def _grid_options_changed(self): if not self._initialized: return self._rebuild_widget() def _show_toolbar_changed(self): if not self._initialized: return self._rebuild_widget() def _update_table(self, update_columns=False, triggered_by=None, scroll_to_row=None, fire_data_change_event=True): df = self._df.copy() from_index = max(self._viewport_range[0] - PAGE_SIZE, 0) to_index = max(self._viewport_range[0] + PAGE_SIZE, 0) new_df_range = (from_index, to_index) if triggered_by is 'viewport_changed' and \ self._df_range == new_df_range: return self._df_range = new_df_range df = df.iloc[from_index:to_index] self._row_count = len(self._df.index) if type(df.index) == pd.core.index.MultiIndex: self._multi_index = True else: self._multi_index = False if update_columns: self._string_columns = list(df.select_dtypes( include=[np.dtype('O'), 'category'] ).columns.values) # call map(str) for all columns identified as string columns, in # case any are not strings already for col_name in self._string_columns: sort_column_name = self._sort_helper_columns.get(col_name) if sort_column_name: series_to_set = df[sort_column_name] else: series_to_set = self._get_col_series_from_df( col_name, df ).map(str) self._set_col_series_on_df(col_name, df, series_to_set) df_json = pd_json.to_json(None, df, orient='table', date_format='iso', double_precision=self.precision) if update_columns: self._interval_columns = [] self._sort_helper_columns = {} self._period_columns = [] # parse the schema that we just exported in order to get the # column metadata that was generated by 'to_json' parsed_json = json.loads(df_json) df_schema = parsed_json['schema'] if ('primaryKey' in df_schema): self._primary_key = df_schema['primaryKey'] else: # for some reason, 'primaryKey' isn't set when the index is # a single interval column. that's why this case is here. self._primary_key = [df.index.name] columns = {} for i, cur_column in enumerate(df_schema['fields']): col_name = cur_column['name'] if 'constraints' in cur_column and \ isinstance(cur_column['constraints']['enum'][0], dict): cur_column['type'] = 'interval' self._interval_columns.append(col_name) if 'freq' in cur_column: self._period_columns.append(col_name) if col_name in self._primary_key: cur_column['is_index'] = True cur_column['position'] = i columns[col_name] = cur_column self._columns = columns # special handling for interval columns: convert to a string column # and then call 'to_json' again to get a new version of the table # json that has interval columns replaced with text columns if len(self._interval_columns) > 0: for col_name in self._interval_columns: col_series = self._get_col_series_from_df(col_name, df) col_series_as_strings = col_series.map(lambda x: str(x)) self._set_col_series_on_df(col_name, df, col_series_as_strings) # special handling for period index columns: call to_timestamp to # convert the series to a datetime series before displaying if len(self._period_columns) > 0: for col_name in self._period_columns: sort_column_name = self._sort_helper_columns.get(col_name) if sort_column_name: series_to_set = df[sort_column_name] else: series_to_set = self._get_col_series_from_df( col_name, df ).to_timestamp() self._set_col_series_on_df(col_name, df, series_to_set) # and then call 'to_json' again to get a new version of the table # json that has interval columns replaced with text columns if len(self._interval_columns) > 0 or len(self._period_columns) > 0: df_json = pd_json.to_json(None, df, orient='table', date_format='iso', double_precision=self.precision) self._df_json = df_json if fire_data_change_event: data_to_send = { 'type': 'update_data_view', 'columns': self._columns, 'triggered_by': triggered_by } if scroll_to_row: data_to_send['scroll_to_row'] = scroll_to_row self.send(data_to_send) def _update_sort(self): try: if self._sort_field == '': return if self._sort_field in self._primary_key: if len(self._primary_key) == 1: self._df.sort_index( ascending=self._sort_ascending, inplace=True ) else: level_id = self._sort_field if self._sort_field.startswith('level_'): level_id = int(self._sort_field[6:]) self._df.sort_index( level=level_id, ascending=self._sort_ascending, inplace=True ) else: self._df.sort_values( self._sort_field, ascending=self._sort_ascending, inplace=True ) except TypeError: self.log.info('TypeError occurred, assuming mixed data type ' 'column') # if there's a TypeError, assume it means that we have a mixed # type column, and attempt to create a stringified version of # the column to use for sorting/filtering self._df.sort_values( self._initialize_sort_column(self._sort_field), ascending=self._sort_ascending, inplace=True ) # Add a new column which is a stringified version of the column whose name # was passed in, which can be used for sorting and filtering (to avoid # error caused by the type of data in the column, like having multiple # data types in a single column). def _initialize_sort_column(self, col_name, to_timestamp=False): sort_column_name = self._sort_helper_columns.get(col_name) if sort_column_name: return sort_column_name sort_col_series = \ self._get_col_series_from_df(col_name, self._df) sort_col_series_unfiltered = \ self._get_col_series_from_df(col_name, self._unfiltered_df) sort_column_name = str(col_name) + self._sort_col_suffix if to_timestamp: self._df[sort_column_name] = sort_col_series.to_timestamp() self._unfiltered_df[sort_column_name] = \ sort_col_series_unfiltered.to_timestamp() else: self._df[sort_column_name] = sort_col_series.map(str) self._unfiltered_df[sort_column_name] = \ sort_col_series_unfiltered.map(str) self._sort_helper_columns[col_name] = sort_column_name return sort_column_name def _handle_get_column_min_max(self, content): col_name = content['field'] col_info = self._columns[col_name] if 'filter_info' in col_info and 'selected' in col_info['filter_info']: df_for_unique = self._unfiltered_df else: df_for_unique = self._df # if there's a period index column, add a sort column which has the # same values, but converted to timestamps instead of period objects. # we'll use that sort column for all subsequent sorts/filters. if col_name in self._period_columns: self._initialize_sort_column(col_name, to_timestamp=True) col_series = self._get_col_series_from_df(col_name, df_for_unique) if 'is_index' in col_info: col_series = pd.Series(col_series) if col_info['type'] in ['integer', 'number']: if 'filter_info' not in col_info or \ (col_info['filter_info']['min'] is None and col_info['filter_info']['max'] is None): col_info['slider_max'] = max(col_series) col_info['slider_min'] = min(col_series) self._columns[col_name] = col_info self.send({ 'type': 'column_min_max_updated', 'field': col_name, 'col_info': col_info }) return elif col_info['type'] == 'datetime': if 'filter_info' not in col_info or \ (col_info['filter_info']['min'] is None and col_info['filter_info']['max'] is None): col_info['filter_max'] = max(col_series) col_info['filter_min'] = min(col_series) self._columns[col_name] = col_info self.send({ 'type': 'column_min_max_updated', 'field': col_name, 'col_info': col_info }) return elif col_info['type'] == 'boolean': self.log.info('handling boolean type') if 'filter_info' not in col_info: values = [] for possible_val in [True, False]: if possible_val in col_series: values.append(possible_val) col_info['values'] = values self._columns[col_name] = col_info self.send({ 'type': 'column_min_max_updated', 'field': col_name, 'col_info': col_info }) self.log.info('handled boolean type') return else: if col_info['type'] == 'any': unique_list = col_series.dtype.categories else: if col_name in self._sorted_column_cache: unique_list = self._sorted_column_cache[col_name] else: unique = col_series.unique() if len(unique) < 500000: try: unique.sort() except TypeError: sort_col_name = \ self._initialize_sort_column(col_name) col_series = df_for_unique[sort_col_name] unique = col_series.unique() unique.sort() unique_list = unique.tolist() self._sorted_column_cache[col_name] = unique_list if content['search_val'] is not None: unique_list = [ k for k in unique_list if content['search_val'].lower() in str(k).lower() ] # if the filter that we're opening is already active (as indicated # by the presence of a 'selected' attribute on the column's # filter_info attribute), show the selected rows at the top and # specify that they should be checked if 'filter_info' in col_info and \ 'selected' in col_info['filter_info']: col_filter_info = col_info['filter_info'] col_filter_table = self._filter_tables[col_name] def get_value_from_filter_table(k): return col_filter_table[k] selected_indices = col_filter_info['selected'] or [] if selected_indices == 'all': excluded_indices = col_filter_info['excluded'] or [] excluded_values = list(map(get_value_from_filter_table, excluded_indices)) non_excluded_count = 0 for i in range(len(unique_list), 0, -1): unique_val = unique_list[i-1] if unique_val not in excluded_values: non_excluded_count += 1 excluded_values.insert(0, unique_val) col_info['values'] = excluded_values col_info['selected_length'] = non_excluded_count elif len(selected_indices) == 0: col_info['selected_length'] = 0 col_info['values'] = unique_list else: selected_vals = list(map(get_value_from_filter_table, selected_indices)) col_info['selected_length'] = len(selected_vals) in_selected = set(selected_vals) in_unique = set(unique_list) in_unique_but_not_selected = list(in_unique - in_selected) in_unique_but_not_selected.sort() selected_vals.extend(in_unique_but_not_selected) col_info['values'] = selected_vals else: col_info['selected_length'] = 0 col_info['values'] = unique_list length = len(col_info['values']) self._filter_tables[col_name] = list(col_info['values']) if col_info['type'] == 'any': col_info['value_range'] = (0, length) else: max_items = PAGE_SIZE * 2 range_max = length if length > max_items: col_info['values'] = col_info['values'][:max_items] range_max = max_items col_info['value_range'] = (0, range_max) col_info['length'] = length self._columns[col_name] = col_info if content['search_val'] is not None: message_type = 'update_data_view_filter' else: message_type = 'column_min_max_updated' try: self.send({ 'type': message_type, 'field': col_name, 'col_info': col_info }) except ValueError: # if there's a ValueError, assume it's because we're # attempting to serialize something that can't be converted # to json, so convert all the values to strings. col_info['values'] = map(str, col_info['values']) self.send({ 'type': message_type, 'field': col_name, 'col_info': col_info }) # get any column from a dataframe, including index columns def _get_col_series_from_df(self, col_name, df): sort_column_name = self._sort_helper_columns.get(col_name) if sort_column_name: return df[sort_column_name] if col_name in self._primary_key: if len(self._primary_key) > 1: key_index = self._primary_key.index(col_name) return df.index.get_level_values(level=key_index) else: return df.index else: return df[col_name] def _set_col_series_on_df(self, col_name, df, col_series): if col_name in self._primary_key: col_series.name = col_name if len(self._primary_key) > 1: key_index = self._primary_key.index(col_name) df.index.set_levels(col_series, level=key_index, inplace=True) else: df.set_index(col_series, inplace=True) else: df[col_name] = col_series def _append_condition_for_column(self, col_name, filter_info, conditions): col_series = self._get_col_series_from_df(col_name, self._unfiltered_df) if filter_info['type'] == 'slider': if filter_info['min'] is not None: conditions.append(col_series >= filter_info['min']) if filter_info['max'] is not None: conditions.append(col_series <= filter_info['max']) elif filter_info['type'] == 'date': if filter_info['min'] is not None: conditions.append( col_series >= pd.to_datetime(filter_info['min'], unit='ms') ) if filter_info['max'] is not None: conditions.append( col_series <= pd.to_datetime(filter_info['max'], unit='ms') ) elif filter_info['type'] == 'boolean': if filter_info['selected'] is not None: conditions.append( col_series == filter_info['selected'] ) elif filter_info['type'] == 'text': if col_name not in self._filter_tables: return col_filter_table = self._filter_tables[col_name] selected_indices = filter_info['selected'] excluded_indices = filter_info['excluded'] def get_value_from_filter_table(i): return col_filter_table[i] if selected_indices == "all": if excluded_indices is not None and len(excluded_indices) > 0: excluded_values = list( map(get_value_from_filter_table, excluded_indices) ) conditions.append(~col_series.isin(excluded_values)) elif selected_indices is not None and len(selected_indices) > 0: selected_values = list( map(get_value_from_filter_table, selected_indices) ) conditions.append(col_series.isin(selected_values)) def _handle_filter_changed(self, content): col_name = content['field'] columns = self._columns.copy() col_info = columns[col_name] col_info['filter_info'] = content['filter_info'] columns[col_name] = col_info conditions = [] for key, value in columns.items(): if 'filter_info' in value: self._append_condition_for_column( key, value['filter_info'], conditions ) self._columns = columns self._ignore_df_changed = True if len(conditions) == 0: self._df = self._unfiltered_df.copy() else: combined_condition = conditions[0] for c in conditions[1:]: combined_condition = combined_condition & c self._df = self._unfiltered_df[combined_condition].copy() if len(self._df) < self._viewport_range[0]: viewport_size = self._viewport_range[1] - self._viewport_range[0] range_top = max(0, len(self._df) - viewport_size) self._viewport_range = (range_top, range_top + viewport_size) self._sorted_column_cache = {} self._update_sort() self._update_table(triggered_by='filter_changed') self._ignore_df_changed = False def _handle_qgrid_msg(self, widget, content, buffers=None): try: self._handle_qgrid_msg_helper(content) except Exception as e: self.log.error(e) self.log.exception("Unhandled exception while handling msg") def _handle_qgrid_msg_helper(self, content): """Handle incoming messages from the QGridView""" if 'type' not in content: return if content['type'] == 'cell_change': col_info = self._columns[content['column']] try: location = (self._df.index[content['row_index']], content['column']) val_to_set = content['value'] if col_info['type'] == 'datetime': val_to_set = pd.to_datetime(val_to_set) self._df.at[location] = val_to_set query = self._unfiltered_df[self._index_col_name] == \ content['unfiltered_index'] self._unfiltered_df.loc[query, content['column']] = val_to_set self._trigger_df_change_event(location) except (ValueError, TypeError): msg = "Error occurred while attempting to edit the " \ "DataFrame. Check the notebook server logs for more " \ "information." self.log.exception(msg) self.send({ 'type': 'show_error', 'error_msg': msg, 'triggered_by': 'add_row' }) return elif content['type'] == 'selection_change': self._selected_rows = content['rows'] elif content['type'] == 'viewport_changed': self._viewport_range = (content['top'], content['bottom']) self._update_table(triggered_by='viewport_changed') elif content['type'] == 'add_row': self.add_row() elif content['type'] == 'remove_row': self.remove_row() elif content['type'] == 'viewport_changed_filter': col_name = content['field'] col_info = self._columns[col_name] col_filter_table = self._filter_tables[col_name] from_index = max(content['top'] - PAGE_SIZE, 0) to_index = max(content['top'] + PAGE_SIZE, 0) col_info['values'] = col_filter_table[from_index:to_index] col_info['value_range'] = (from_index, to_index) self._columns[col_name] = col_info self.send({ 'type': 'update_data_view_filter', 'field': col_name, 'col_info': col_info }) elif content['type'] == 'sort_changed': self._sort_field = content['sort_field'] self._sort_ascending = content['sort_ascending'] self._sorted_column_cache = {} self._update_sort() self._update_table(triggered_by='sort_changed') self._trigger_df_change_event() elif content['type'] == 'get_column_min_max': self._handle_get_column_min_max(content) elif content['type'] == 'filter_changed': self._handle_filter_changed(content) def _trigger_df_change_event(self, location=None): self.notify_change(Bunch( name='_df', old=None, new=self._df, owner=self, type='change', location=location, )) def get_changed_df(self): """ Get a copy of the DataFrame that was used to create the current instance of QgridWidget which reflects the current state of the UI. This includes any sorting or filtering changes, as well as edits that have been made by double clicking cells. :rtype: DataFrame """ col_names_to_drop = list(self._sort_helper_columns.values()) col_names_to_drop.append(self._index_col_name) return self._df.drop(col_names_to_drop, axis=1) def get_selected_df(self): """ Get a DataFrame which reflects the current state of the UI and only includes the currently selected row(s). Internally it calls ``get_changed_df()`` and then filters down to the selected rows using ``iloc``. :rtype: DataFrame """ changed_df = self.get_changed_df() return changed_df.iloc[self._selected_rows] def get_selected_rows(self): """ Get the currently selected rows. :rtype: List of integers """ return self._selected_rows def add_row(self): """ Append a row at the end of the dataframe by duplicating the last row and incrementing it's index by 1. The feature is only available for DataFrames that have an integer index. """ df = self._df if not df.index.is_integer(): msg = "Cannot add a row to a table with a non-integer index" self.send({ 'type': 'show_error', 'error_msg': msg, 'triggered_by': 'add_row' }) return last_index = max(df.index) last = df.loc[last_index].copy() last.name += 1 last[self._index_col_name] = last.name df.loc[last.name] = last.values self._unfiltered_df.loc[last.name] = last.values self._update_table(triggered_by='add_row', scroll_to_row=df.index.get_loc(last.name)) self._trigger_df_change_event() def remove_row(self): """ Remove the current row from the table. """ if self._multi_index: msg = "Cannot remove a row from a table with a multi index" self.send({ 'type': 'show_error', 'error_msg': msg, 'triggered_by': 'remove_row' }) return selected_names = \ map(lambda x: self._df.iloc[x].name, self._selected_rows) self._df.drop(selected_names, inplace=True) self._unfiltered_df.drop(selected_names, inplace=True) self._selected_rows = [] self._update_table(triggered_by='remove_row') self._trigger_df_change_event()
class SingleUserNotebookApp(LabApp): """A Subclass of the regular LabApp that is aware of the parent multiuser context.""" description = dedent(""" Single-user server for JupyterHub. Extends the Jupyter Notebook server. Meant to be invoked by JupyterHub Spawners, and not directly. """) examples = "" subcommands = {} version = __version__ classes = LabApp.classes + [HubOAuth] # disable single-user app's localhost checking allow_remote_access = True # don't store cookie secrets cookie_secret_file = '' # always generate a new cookie secret on launch # ensures that each spawn clears any cookies from previous session, # triggering OAuth again cookie_secret = Bytes() def _cookie_secret_default(self): return os.urandom(32) user = CUnicode().tag(config=True) group = CUnicode().tag(config=True) @default('user') def _default_user(self): return os.environ.get('JUPYTERHUB_USER') or '' @default('group') def _default_group(self): return os.environ.get('JUPYTERHUB_GROUP') or '' @observe('user') def _user_changed(self, change): self.log.name = change.new hub_host = Unicode().tag(config=True) hub_prefix = Unicode('/hub/').tag(config=True) @default('keyfile') def _keyfile_default(self): return os.environ.get('JUPYTERHUB_SSL_KEYFILE') or '' @default('certfile') def _certfile_default(self): return os.environ.get('JUPYTERHUB_SSL_CERTFILE') or '' @default('client_ca') def _client_ca_default(self): return os.environ.get('JUPYTERHUB_SSL_CLIENT_CA') or '' @default('hub_prefix') def _hub_prefix_default(self): base_url = os.environ.get('JUPYTERHUB_BASE_URL') or '/' return base_url + 'hub/' hub_api_url = Unicode().tag(config=True) @default('hub_api_url') def _hub_api_url_default(self): return os.environ.get( 'JUPYTERHUB_API_URL') or 'http://127.0.0.1:8081/hub/api' # defaults for some configurables that may come from service env variables: @default('base_url') def _base_url_default(self): return os.environ.get('JUPYTERHUB_SERVICE_PREFIX') or '/' # Note: this may be removed if notebook module is >= 5.0.0b1 @validate('base_url') def _validate_base_url(self, proposal): """ensure base_url starts and ends with /""" value = proposal.value if not value.startswith('/'): value = '/' + value if not value.endswith('/'): value = value + '/' return value @default('port') def _port_default(self): if os.environ.get('JUPYTERHUB_SERVICE_URL'): url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL']) if url.port: return url.port elif url.scheme == 'http': return 80 elif url.scheme == 'https': return 443 return 8888 @default('ip') def _ip_default(self): if os.environ.get('JUPYTERHUB_SERVICE_URL'): url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL']) if url.hostname: return url.hostname return '127.0.0.1' aliases = aliases flags = flags # disble some single-user configurables token = '' open_browser = False quit_button = False trust_xheaders = True login_handler_class = JupyterHubLoginHandler logout_handler_class = JupyterHubLogoutHandler port_retries = ( 0 # disable port-retries, since the Spawner will tell us what port to use ) disable_user_config = Bool( False, help="""Disable user configuration of single-user server. Prevents user-writable files that normally configure the single-user server from being loaded, ensuring admins have full control of configuration. """, ).tag(config=True) @validate('notebook_dir') def _notebook_dir_validate(self, proposal): value = os.path.expanduser(proposal['value']) # Strip any trailing slashes # *except* if it's root _, path = os.path.splitdrive(value) if path == os.sep: return value value = value.rstrip(os.sep) if not os.path.isabs(value): # If we receive a non-absolute path, make it absolute. value = os.path.abspath(value) if not os.path.isdir(value): raise TraitError("No such notebook dir: %r" % value) return value @default('log_datefmt') def _log_datefmt_default(self): """Exclude date from default date format""" return "%Y-%m-%d %H:%M:%S" @default('log_format') def _log_format_default(self): """override default log format to include time""" return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s" def _confirm_exit(self): # disable the exit confirmation for background notebook processes self.io_loop.add_callback_from_signal(self.io_loop.stop) def migrate_config(self): if self.disable_user_config: # disable config-migration when user config is disabled return else: super(SingleUserNotebookApp, self).migrate_config() @property def config_file_paths(self): path = super(SingleUserNotebookApp, self).config_file_paths if self.disable_user_config: # filter out user-writable config dirs if user config is disabled path = list(_exclude_home(path)) return path @property def nbextensions_path(self): path = super(SingleUserNotebookApp, self).nbextensions_path if self.disable_user_config: path = list(_exclude_home(path)) return path @validate('static_custom_path') def _validate_static_custom_path(self, proposal): path = proposal['value'] if self.disable_user_config: path = list(_exclude_home(path)) return path # create dynamic default http client, # configured with any relevant ssl config hub_http_client = Any() @default('hub_http_client') def _default_client(self): ssl_context = make_ssl_context(self.keyfile, self.certfile, cafile=self.client_ca) AsyncHTTPClient.configure(None, defaults={"ssl_options": ssl_context}) return AsyncHTTPClient() async def check_hub_version(self): """Test a connection to my Hub - exit if I can't connect at all - check version and warn on sufficient mismatch """ client = self.hub_http_client RETRIES = 5 for i in range(1, RETRIES + 1): try: resp = await client.fetch(self.hub_api_url) except Exception: self.log.exception( "Failed to connect to my Hub at %s (attempt %i/%i). Is it running?", self.hub_api_url, i, RETRIES, ) await gen.sleep(min(2**i, 16)) else: break else: self.exit(1) hub_version = resp.headers.get('X-JupyterHub-Version') _check_version(hub_version, __version__, self.log) server_name = Unicode() @default('server_name') def _server_name_default(self): return os.environ.get('JUPYTERHUB_SERVER_NAME', '') hub_activity_url = Unicode( config=True, help="URL for sending JupyterHub activity updates") @default('hub_activity_url') def _default_activity_url(self): return os.environ.get('JUPYTERHUB_ACTIVITY_URL', '') hub_activity_interval = Integer( 300, config=True, help=""" Interval (in seconds) on which to update the Hub with our latest activity. """, ) @default('hub_activity_interval') def _default_activity_interval(self): env_value = os.environ.get('JUPYTERHUB_ACTIVITY_INTERVAL') if env_value: return int(env_value) else: return 300 _last_activity_sent = Any(allow_none=True) async def notify_activity(self): """Notify jupyterhub of activity""" client = self.hub_http_client last_activity = self.web_app.last_activity() if not last_activity: self.log.debug("No activity to send to the Hub") return if last_activity: # protect against mixed timezone comparisons if not last_activity.tzinfo: # assume naive timestamps are utc self.log.warning("last activity is using naive timestamps") last_activity = last_activity.replace(tzinfo=timezone.utc) if self._last_activity_sent and last_activity < self._last_activity_sent: self.log.debug("No activity since %s", self._last_activity_sent) return last_activity_timestamp = isoformat(last_activity) async def notify(): self.log.debug("Notifying Hub of activity %s", last_activity_timestamp) req = HTTPRequest( url=self.hub_activity_url, method='POST', headers={ "Authorization": "token {}".format(self.hub_auth.api_token), "Content-Type": "application/json", }, body=json.dumps({ 'servers': { self.server_name: { 'last_activity': last_activity_timestamp } }, 'last_activity': last_activity_timestamp, }), ) try: await client.fetch(req) except Exception: self.log.exception("Error notifying Hub of activity") return False else: return True await exponential_backoff( notify, fail_message="Failed to notify Hub of activity", start_wait=1, max_wait=15, timeout=60, ) self._last_activity_sent = last_activity async def keep_activity_updated(self): if not self.hub_activity_url or not self.hub_activity_interval: self.log.warning("Activity events disabled") return self.log.info("Updating Hub with activity every %s seconds", self.hub_activity_interval) while True: try: await self.notify_activity() except Exception as e: self.log.exception("Error notifying Hub of activity") # add 20% jitter to the interval to avoid alignment # of lots of requests from user servers t = self.hub_activity_interval * (1 + 0.2 * (random.random() - 0.5)) await asyncio.sleep(t) def initialize(self, argv=None): # disable trash by default # this can be re-enabled by config self.config.FileContentsManager.delete_to_trash = False return super().initialize(argv) def start(self): self.log.info("Starting jupyterhub-singleuser server version %s", __version__) # start by hitting Hub to check version ioloop.IOLoop.current().run_sync(self.check_hub_version) ioloop.IOLoop.current().add_callback(self.keep_activity_updated) super(SingleUserNotebookApp, self).start() def init_hub_auth(self): api_token = None if os.getenv('JPY_API_TOKEN'): # Deprecated env variable (as of 0.7.2) api_token = os.environ['JPY_API_TOKEN'] if os.getenv('JUPYTERHUB_API_TOKEN'): api_token = os.environ['JUPYTERHUB_API_TOKEN'] if not api_token: self.exit( "JUPYTERHUB_API_TOKEN env is required to run jupyterhub-singleuser. Did you launch it manually?" ) self.hub_auth = HubOAuth( parent=self, api_token=api_token, api_url=self.hub_api_url, hub_prefix=self.hub_prefix, base_url=self.base_url, keyfile=self.keyfile, certfile=self.certfile, client_ca=self.client_ca, ) # smoke check if not self.hub_auth.oauth_client_id: raise ValueError("Missing OAuth client ID") def init_webapp(self): # load the hub-related settings into the tornado settings dict self.init_hub_auth() s = self.tornado_settings s['log_function'] = log_request s['user'] = self.user s['group'] = self.group s['hub_prefix'] = self.hub_prefix s['hub_host'] = self.hub_host s['hub_auth'] = self.hub_auth csp_report_uri = s[ 'csp_report_uri'] = self.hub_host + url_path_join( self.hub_prefix, 'security/csp-report') headers = s.setdefault('headers', {}) headers['X-JupyterHub-Version'] = __version__ # set CSP header directly to workaround bugs in jupyter/notebook 5.0 headers.setdefault( 'Content-Security-Policy', ';'.join( ["frame-ancestors 'self'", "report-uri " + csp_report_uri]), ) super(SingleUserNotebookApp, self).init_webapp() # add OAuth callback self.web_app.add_handlers( r".*$", [(urlparse(self.hub_auth.oauth_redirect_uri).path, OAuthCallbackHandler)], ) # apply X-JupyterHub-Version to *all* request handlers (even redirects) self.patch_default_headers() self.patch_templates() def patch_default_headers(self): if hasattr(RequestHandler, '_orig_set_default_headers'): return RequestHandler._orig_set_default_headers = RequestHandler.set_default_headers def set_jupyterhub_header(self): self._orig_set_default_headers() self.set_header('X-JupyterHub-Version', __version__) RequestHandler.set_default_headers = set_jupyterhub_header def patch_templates(self): """Patch page templates to add Hub-related buttons""" self.jinja_template_vars[ 'logo_url'] = self.hub_host + url_path_join( self.hub_prefix, 'logo') self.jinja_template_vars['hub_host'] = self.hub_host self.jinja_template_vars['hub_prefix'] = self.hub_prefix env = self.web_app.settings['jinja2_env'] env.globals[ 'hub_control_panel_url'] = self.hub_host + url_path_join( self.hub_prefix, 'home') # patch jinja env loading to modify page template def get_page(name): if name == 'page.html': return page_template orig_loader = env.loader env.loader = ChoiceLoader([FunctionLoader(get_page), orig_loader])
class MultiKernelManager(LoggingConfigurable): """A class for managing multiple kernels.""" default_kernel_name = Unicode( NATIVE_KERNEL_NAME, config=True, help="The name of the default kernel to start") kernel_spec_manager = Instance(KernelSpecManager, allow_none=True) kernel_manager_class = DottedObjectName( "jupyter_client.ioloop.IOLoopKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the KernelManager for customized behavior. """) def __init__(self, *args, **kwargs): super(MultiKernelManager, self).__init__(*args, **kwargs) # Cache all the currently used ports self.currently_used_ports = set() @observe('kernel_manager_class') def _kernel_manager_class_changed(self, change): self.kernel_manager_factory = self._create_kernel_manager_factory() kernel_manager_factory = Any( help="this is kernel_manager_class after import") @default('kernel_manager_factory') def _kernel_manager_factory_default(self): return self._create_kernel_manager_factory() def _create_kernel_manager_factory(self): kernel_manager_ctor = import_item(self.kernel_manager_class) def create_kernel_manager(*args, **kwargs): if self.shared_context: if self.context.closed: # recreate context if closed self.context = self._context_default() kwargs.setdefault("context", self.context) km = kernel_manager_ctor(*args, **kwargs) if km.cache_ports: km.shell_port = self._find_available_port(km.ip) km.iopub_port = self._find_available_port(km.ip) km.stdin_port = self._find_available_port(km.ip) km.hb_port = self._find_available_port(km.ip) km.control_port = self._find_available_port(km.ip) return km return create_kernel_manager def _find_available_port(self, ip): while True: tmp_sock = socket.socket() tmp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) tmp_sock.bind((ip, 0)) port = tmp_sock.getsockname()[1] tmp_sock.close() # This is a workaround for https://github.com/jupyter/jupyter_client/issues/487 # We prevent two kernels to have the same ports. if port not in self.currently_used_ports: self.currently_used_ports.add(port) return port shared_context = Bool( True, config=True, help="Share a single zmq.Context to talk to all my kernels", ) _created_context = Bool(False) context = Instance('zmq.Context') @default("context") def _context_default(self): self._created_context = True return zmq.Context() def __del__(self): if self._created_context and self.context and not self.context.closed: if self.log: self.log.debug("Destroying zmq context for %s", self) self.context.destroy() try: super_del = super().__del__ except AttributeError: pass else: super_del() connection_dir = Unicode('') _kernels = Dict() def list_kernel_ids(self): """Return a list of the kernel ids of the active kernels.""" # Create a copy so we can iterate over kernels in operations # that delete keys. return list(self._kernels.keys()) def __len__(self): """Return the number of running kernels.""" return len(self.list_kernel_ids()) def __contains__(self, kernel_id): return kernel_id in self._kernels def pre_start_kernel(self, kernel_name, kwargs): # kwargs should be mutable, passing it as a dict argument. kernel_id = kwargs.pop('kernel_id', self.new_kernel_id(**kwargs)) if kernel_id in self: raise DuplicateKernelError('Kernel already exists: %s' % kernel_id) if kernel_name is None: kernel_name = self.default_kernel_name # kernel_manager_factory is the constructor for the KernelManager # subclass we are using. It can be configured as any Configurable, # including things like its transport and ip. constructor_kwargs = {} if self.kernel_spec_manager: constructor_kwargs[ 'kernel_spec_manager'] = self.kernel_spec_manager km = self.kernel_manager_factory(connection_file=os.path.join( self.connection_dir, "kernel-%s.json" % kernel_id), parent=self, log=self.log, kernel_name=kernel_name, **constructor_kwargs) return km, kernel_name, kernel_id def start_kernel(self, kernel_name=None, **kwargs): """Start a new kernel. The caller can pick a kernel_id by passing one in as a keyword arg, otherwise one will be generated using new_kernel_id(). The kernel ID for the newly started kernel is returned. """ km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs) km.start_kernel(**kwargs) self._kernels[kernel_id] = km return kernel_id def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by its kernel uuid. Parameters ========== kernel_id : uuid The id of the kernel to shutdown. now : bool Should the kernel be shutdown forcibly using a signal. restart : bool Will the kernel be restarted? """ self.log.info("Kernel shutdown: %s" % kernel_id) km = self.get_kernel(kernel_id) ports = (km.shell_port, km.iopub_port, km.stdin_port, km.hb_port, km.control_port) km.shutdown_kernel(now=now, restart=restart) self.remove_kernel(kernel_id) if km.cache_ports and not restart: for port in ports: self.currently_used_ports.remove(port) @kernel_method def request_shutdown(self, kernel_id, restart=False): """Ask a kernel to shut down by its kernel uuid""" @kernel_method def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1): """Wait for a kernel to finish shutting down, and kill it if it doesn't """ self.log.info("Kernel shutdown: %s" % kernel_id) @kernel_method def cleanup(self, kernel_id, connection_file=True): """Clean up a kernel's resources""" @kernel_method def cleanup_resources(self, kernel_id, restart=False): """Clean up a kernel's resources""" def remove_kernel(self, kernel_id): """remove a kernel from our mapping. Mainly so that a kernel can be removed if it is already dead, without having to call shutdown_kernel. The kernel object is returned. """ return self._kernels.pop(kernel_id) def shutdown_all(self, now=False): """Shutdown all kernels.""" kids = self.list_kernel_ids() for kid in kids: self.request_shutdown(kid) for kid in kids: self.finish_shutdown(kid) self.cleanup(kid) self.remove_kernel(kid) @kernel_method def interrupt_kernel(self, kernel_id): """Interrupt (SIGINT) the kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel interrupted: %s" % kernel_id) @kernel_method def signal_kernel(self, kernel_id, signum): """Sends a signal to the kernel by its uuid. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. Parameters ========== kernel_id : uuid The id of the kernel to signal. """ self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @kernel_method def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters ========== kernel_id : uuid The id of the kernel to interrupt. """ self.log.info("Kernel restarted: %s" % kernel_id) @kernel_method def is_alive(self, kernel_id): """Is the kernel alive. This calls KernelManager.is_alive() which calls Popen.poll on the actual kernel subprocess. Parameters ========== kernel_id : uuid The id of the kernel. """ def _check_kernel_id(self, kernel_id): """check that a kernel id is valid""" if kernel_id not in self: raise KeyError("Kernel with id not found: %s" % kernel_id) def get_kernel(self, kernel_id): """Get the single KernelManager object for a kernel by its uuid. Parameters ========== kernel_id : uuid The id of the kernel. """ self._check_kernel_id(kernel_id) return self._kernels[kernel_id] @kernel_method def add_restart_callback(self, kernel_id, callback, event='restart'): """add a callback for the KernelRestarter""" @kernel_method def remove_restart_callback(self, kernel_id, callback, event='restart'): """remove a callback for the KernelRestarter""" @kernel_method def get_connection_info(self, kernel_id): """Return a dictionary of connection data for a kernel. Parameters ========== kernel_id : uuid The id of the kernel. Returns ======= connection_dict : dict A dict of the information needed to connect to a kernel. This includes the ip address and the integer port numbers of the different channels (stdin_port, iopub_port, shell_port, hb_port). """ @kernel_method def connect_iopub(self, kernel_id, identity=None): """Return a zmq Socket connected to the iopub channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_shell(self, kernel_id, identity=None): """Return a zmq Socket connected to the shell channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_control(self, kernel_id, identity=None): """Return a zmq Socket connected to the control channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_stdin(self, kernel_id, identity=None): """Return a zmq Socket connected to the stdin channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ @kernel_method def connect_hb(self, kernel_id, identity=None): """Return a zmq Socket connected to the hb channel. Parameters ========== kernel_id : uuid The id of the kernel identity : bytes (optional) The zmq identity of the socket Returns ======= stream : zmq Socket or ZMQStream """ def new_kernel_id(self, **kwargs): """ Returns the id to associate with the kernel for this request. Subclasses may override this method to substitute other sources of kernel ids. :param kwargs: :return: string-ized version 4 uuid """ return unicode_type(uuid.uuid4())
class ZMQInteractiveShell(InteractiveShell): """A subclass of InteractiveShell for ZMQ.""" displayhook_class = Type(ZMQShellDisplayHook) display_pub_class = Type(ZMQDisplayPublisher) data_pub_class = Type('ibpykernel.datapub.ZMQDataPublisher') kernel = Any() parent_header = Any() @default('banner1') def _default_banner1(self): return brackets.shell.BANNER # Override the traitlet in the parent class, because there's no point using # readline for the kernel. Can be removed when the readline code is moved # to the terminal frontend. colors_force = CBool(True) readline_use = CBool(False) # autoindent has no meaning in a zmqshell, and attempting to enable it # will print a warning in the absence of readline. autoindent = CBool(False) exiter = Instance(ZMQExitAutocall) @default('exiter') def _default_exiter(self): return ZMQExitAutocall(self) @observe('exit_now') def _update_exit_now(self, change): """stop eventloop when exit_now fires""" if change['new']: loop = ioloop.IOLoop.instance() loop.add_timeout(time.time() + 0.1, loop.stop) keepkernel_on_exit = None # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no # interactive input being read; we provide event loop support in ipkernel def enable_gui(self, gui): from .eventloops import enable_gui as real_enable_gui try: real_enable_gui(gui) self.active_eventloop = gui except ValueError as e: raise UsageError("%s" % e) def init_environment(self): """Configure the user's environment.""" env = os.environ # These two ensure 'ls' produces nice coloring on BSD-derived systems env['TERM'] = 'xterm-color' env['CLICOLOR'] = '1' # Since normal pagers don't work at all (over pexpect we don't have # single-key control of the subprocess), try to disable paging in # subprocesses as much as possible. env['PAGER'] = 'cat' env['GIT_PAGER'] = 'cat' def init_hooks(self): super(ZMQInteractiveShell, self).init_hooks() self.set_hook('show_in_pager', page.as_hook(payloadpage.page), 99) def init_data_pub(self): """Delay datapub init until request, for deprecation warnings""" pass @property def data_pub(self): if not hasattr(self, '_data_pub'): warnings.warn( "InteractiveShell.data_pub is deprecated outside IPython parallel.", DeprecationWarning, stacklevel=2) self._data_pub = self.data_pub_class(parent=self) self._data_pub.session = self.display_pub.session self._data_pub.pub_socket = self.display_pub.pub_socket return self._data_pub @data_pub.setter def data_pub(self, pub): self._data_pub = pub def ask_exit(self): """Engage the exit actions.""" self.exit_now = (not self.keepkernel_on_exit) payload = dict( source='ask_exit', keepkernel=self.keepkernel_on_exit, ) self.payload_manager.write_payload(payload) def run_cell(self, *args, **kwargs): self._last_traceback = None return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs) def _showtraceback(self, etype, evalue, stb): # try to preserve ordering of tracebacks and print statements sys.stdout.flush() sys.stderr.flush() exc_content = { u'traceback': stb, u'ename': unicode_type(etype.__name__), u'evalue': py3compat.safe_unicode(evalue), } dh = self.displayhook # Send exception info over pub socket for other clients than the caller # to pick up topic = None if dh.topic: topic = dh.topic.replace(b'execute_result', b'error') exc_msg = dh.session.send(dh.pub_socket, u'error', json_clean(exc_content), dh.parent_header, ident=topic) # FIXME - Once we rely on Python 3, the traceback is stored on the # exception object, so we shouldn't need to store it here. self._last_traceback = stb def set_next_input(self, text, replace=False): """Send the specified text to the frontend to be presented at the next input cell.""" payload = dict( source='set_next_input', text=text, replace=replace, ) self.payload_manager.write_payload(payload) def set_parent(self, parent): """Set the parent header for associating output with its triggering input""" self.parent_header = parent self.displayhook.set_parent(parent) self.display_pub.set_parent(parent) if hasattr(self, '_data_pub'): self.data_pub.set_parent(parent) try: sys.stdout.set_parent(parent) except AttributeError: pass try: sys.stderr.set_parent(parent) except AttributeError: pass def get_parent(self): return self.parent_header def init_magics(self): super(ZMQInteractiveShell, self).init_magics() self.register_magics(KernelMagics) self.magics_manager.register_alias('ed', 'edit') def init_virtualenv(self): # Overridden not to do virtualenv detection, because it's probably # not appropriate in a kernel. To use a kernel in a virtualenv, install # it inside the virtualenv. # https://ipython.readthedocs.io/en/latest/install/kernel_install.html pass
class MappingKernelManager(MultiKernelManager): """A KernelManager that handles - File mapping - HTTP error handling - Kernel message filtering """ @default('kernel_manager_class') def _default_kernel_manager_class(self): return "jupyter_client.ioloop.IOLoopKernelManager" kernel_argv = List(Unicode()) root_dir = Unicode(config=True) _kernel_connections = Dict() _kernel_ports = Dict() _culler_callback = None _initialized_culler = False @default('root_dir') def _default_root_dir(self): try: return self.parent.root_dir except AttributeError: return os.getcwd() @validate('root_dir') def _update_root_dir(self, proposal): """Do a bit of validation of the root dir.""" value = proposal['value'] if not os.path.isabs(value): # If we receive a non-absolute path, make it absolute. value = os.path.abspath(value) if not exists(value) or not os.path.isdir(value): raise TraitError("kernel root dir %r is not a directory" % value) return value cull_idle_timeout = Integer( 0, config=True, help= """Timeout (in seconds) after which a kernel is considered idle and ready to be culled. Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled for users with poor network connections.""") cull_interval_default = 300 # 5 minutes cull_interval = Integer( cull_interval_default, config=True, help= """The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value.""" ) cull_connected = Bool( False, config=True, help= """Whether to consider culling kernels which have one or more connections. Only effective if cull_idle_timeout > 0.""") cull_busy = Bool( False, config=True, help="""Whether to consider culling kernels which are busy. Only effective if cull_idle_timeout > 0.""") buffer_offline_messages = Bool( True, config=True, help= """Whether messages from kernels whose frontends have disconnected should be buffered in-memory. When True (default), messages are buffered and replayed on reconnect, avoiding lost messages due to interrupted connectivity. Disable if long-running kernels will produce too much output while no frontends are connected. """) kernel_info_timeout = Float( 60, config=True, help="""Timeout for giving up on a kernel (in seconds). On starting and restarting kernels, we check whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take before being presumed dead. This affects the MappingKernelManager (which handles kernel restarts) and the ZMQChannelsHandler (which handles the startup). """) _kernel_buffers = Any() @default('_kernel_buffers') def _default_kernel_buffers(self): return defaultdict(lambda: { 'buffer': [], 'session_key': '', 'channels': {} }) last_kernel_activity = Instance( datetime, help="The last activity on any kernel, including shutting down a kernel" ) def __init__(self, **kwargs): self.pinned_superclass = MultiKernelManager self.pinned_superclass.__init__(self, **kwargs) self.last_kernel_activity = utcnow() allowed_message_types = List( trait=Unicode(), config=True, help="""White list of allowed kernel message types. When the list is empty, all message types are allowed. """) allow_tracebacks = Bool( True, config=True, help=('Whether to send tracebacks to clients on exceptions.')) traceback_replacement_message = Unicode( 'An exception occurred at runtime, which is not shown due to security reasons.', config=True, help= ('Message to print when allow_tracebacks is False, and an exception occurs' )) #------------------------------------------------------------------------- # Methods for managing kernels and sessions #------------------------------------------------------------------------- def _handle_kernel_died(self, kernel_id): """notice that a kernel died""" self.log.warning("Kernel %s died, removing from map.", kernel_id) self.remove_kernel(kernel_id) def cwd_for_path(self, path): """Turn API path into absolute OS path.""" os_path = to_os_path(path, self.root_dir) # in the case of documents and kernels not being on the same filesystem, # walk up to root_dir if the paths don't exist while not os.path.isdir(os_path) and os_path != self.root_dir: os_path = os.path.dirname(os_path) return os_path async def start_kernel(self, kernel_id=None, path=None, **kwargs): """Start a kernel for a session and return its kernel_id. Parameters ---------- kernel_id : uuid The uuid to associate the new kernel with. If this is not None, this kernel will be persistent whenever it is requested. path : API path The API path (unicode, '/' delimited) for the cwd. Will be transformed to an OS path relative to root_dir. kernel_name : str The name identifying which kernel spec to launch. This is ignored if an existing kernel is returned, but it may be checked in the future. """ if kernel_id is None or kernel_id not in self: if path is not None: kwargs['cwd'] = self.cwd_for_path(path) if kernel_id is not None: kwargs['kernel_id'] = kernel_id kernel_id = await ensure_async( self.pinned_superclass.start_kernel(self, **kwargs)) self._kernel_connections[kernel_id] = 0 self._kernel_ports[kernel_id] = self._kernels[kernel_id].ports self.start_watching_activity(kernel_id) self.log.info("Kernel started: %s" % kernel_id) self.log.debug("Kernel args: %r" % kwargs) # register callback for failed auto-restart self.add_restart_callback( kernel_id, lambda: self._handle_kernel_died(kernel_id), 'dead', ) # Increase the metric of number of kernels running # for the relevant kernel type by 1 KERNEL_CURRENTLY_RUNNING_TOTAL.labels( type=self._kernels[kernel_id].kernel_name).inc() else: self.log.info("Using existing kernel: %s" % kernel_id) # Initialize culling if not already if not self._initialized_culler: self.initialize_culler() return kernel_id def ports_changed(self, kernel_id): """Used by ZMQChannelsHandler to determine how to coordinate nudge and replays. Ports are captured when starting a kernel (via MappingKernelManager). Ports are considered changed (following restarts) if the referenced KernelManager is using a set of ports different from those captured at startup. If changes are detected, the captured set is updated and a value of True is returned. NOTE: Use is exclusive to ZMQChannelsHandler because this object is a singleton instance while ZMQChannelsHandler instances are per WebSocket connection that can vary per kernel lifetime. """ changed_ports = self._get_changed_ports(kernel_id) if changed_ports: # If changed, update captured ports and return True, else return False. self.log.debug(f"Port change detected for kernel: {kernel_id}") self._kernel_ports[kernel_id] = changed_ports return True return False def _get_changed_ports(self, kernel_id): """Internal method to test if a kernel's ports have changed and, if so, return their values. This method does NOT update the captured ports for the kernel as that can only be done by ZMQChannelsHandler, but instead returns the new list of ports if they are different than those captured at startup. This enables the ability to conditionally restart activity monitoring immediately following a kernel's restart (if ports have changed). """ # Get current ports and return comparison with ports captured at startup. km = self.get_kernel(kernel_id) if km.ports != self._kernel_ports[kernel_id]: return km.ports return None def start_buffering(self, kernel_id, session_key, channels): """Start buffering messages for a kernel Parameters ---------- kernel_id : str The id of the kernel to stop buffering. session_key : str The session_key, if any, that should get the buffer. If the session_key matches the current buffered session_key, the buffer will be returned. channels : dict({'channel': ZMQStream}) The zmq channels whose messages should be buffered. """ if not self.buffer_offline_messages: for channel, stream in channels.items(): stream.close() return self.log.info("Starting buffering for %s", session_key) self._check_kernel_id(kernel_id) # clear previous buffering state self.stop_buffering(kernel_id) buffer_info = self._kernel_buffers[kernel_id] # record the session key because only one session can buffer buffer_info['session_key'] = session_key # TODO: the buffer should likely be a memory bounded queue, we're starting with a list to keep it simple buffer_info['buffer'] = [] buffer_info['channels'] = channels # forward any future messages to the internal buffer def buffer_msg(channel, msg_parts): self.log.debug("Buffering msg on %s:%s", kernel_id, channel) buffer_info['buffer'].append((channel, msg_parts)) for channel, stream in channels.items(): stream.on_recv(partial(buffer_msg, channel)) def get_buffer(self, kernel_id, session_key): """Get the buffer for a given kernel Parameters ---------- kernel_id : str The id of the kernel to stop buffering. session_key : str, optional The session_key, if any, that should get the buffer. If the session_key matches the current buffered session_key, the buffer will be returned. """ self.log.debug("Getting buffer for %s", kernel_id) if kernel_id not in self._kernel_buffers: return buffer_info = self._kernel_buffers[kernel_id] if buffer_info['session_key'] == session_key: # remove buffer self._kernel_buffers.pop(kernel_id) # only return buffer_info if it's a match return buffer_info else: self.stop_buffering(kernel_id) def stop_buffering(self, kernel_id): """Stop buffering kernel messages Parameters ---------- kernel_id : str The id of the kernel to stop buffering. """ self.log.debug("Clearing buffer for %s", kernel_id) self._check_kernel_id(kernel_id) if kernel_id not in self._kernel_buffers: return buffer_info = self._kernel_buffers.pop(kernel_id) # close buffering streams for stream in buffer_info['channels'].values(): if not stream.closed(): stream.on_recv(None) stream.close() msg_buffer = buffer_info['buffer'] if msg_buffer: self.log.info("Discarding %s buffered messages for %s", len(msg_buffer), buffer_info['session_key']) def shutdown_kernel(self, kernel_id, now=False, restart=False): """Shutdown a kernel by kernel_id""" self._check_kernel_id(kernel_id) self.stop_watching_activity(kernel_id) self.stop_buffering(kernel_id) self._kernel_connections.pop(kernel_id, None) # Decrease the metric of number of kernels # running for the relevant kernel type by 1 KERNEL_CURRENTLY_RUNNING_TOTAL.labels( type=self._kernels[kernel_id].kernel_name).dec() self.pinned_superclass.shutdown_kernel(self, kernel_id, now=now, restart=restart) # Unlike its async sibling method in AsyncMappingKernelManager, removing the kernel_id # from the connections dictionary isn't as problematic before the shutdown since the # method is synchronous. However, we'll keep the relative call orders the same from # a maintenance perspective. self._kernel_connections.pop(kernel_id, None) self._kernel_ports.pop(kernel_id, None) async def restart_kernel(self, kernel_id, now=False): """Restart a kernel by kernel_id""" self._check_kernel_id(kernel_id) await ensure_async( self.pinned_superclass.restart_kernel(self, kernel_id, now=now)) kernel = self.get_kernel(kernel_id) # return a Future that will resolve when the kernel has successfully restarted channel = kernel.connect_shell() future = Future() def finish(): """Common cleanup when restart finishes/fails for any reason.""" if not channel.closed(): channel.close() loop.remove_timeout(timeout) kernel.remove_restart_callback(on_restart_failed, 'dead') def on_reply(msg): self.log.debug("Kernel info reply received: %s", kernel_id) finish() if not future.done(): future.set_result(msg) def on_timeout(): self.log.warning("Timeout waiting for kernel_info_reply: %s", kernel_id) finish() if not future.done(): future.set_exception( TimeoutError("Timeout waiting for restart")) def on_restart_failed(): self.log.warning("Restarting kernel failed: %s", kernel_id) finish() if not future.done(): future.set_exception(RuntimeError("Restart failed")) kernel.add_restart_callback(on_restart_failed, 'dead') kernel.session.send(channel, "kernel_info_request") channel.on_recv(on_reply) loop = IOLoop.current() timeout = loop.add_timeout(loop.time() + self.kernel_info_timeout, on_timeout) # Re-establish activity watching if ports have changed... if self._get_changed_ports(kernel_id) is not None: self.stop_watching_activity(kernel_id) self.start_watching_activity(kernel_id) return future def notify_connect(self, kernel_id): """Notice a new connection to a kernel""" if kernel_id in self._kernel_connections: self._kernel_connections[kernel_id] += 1 def notify_disconnect(self, kernel_id): """Notice a disconnection from a kernel""" if kernel_id in self._kernel_connections: self._kernel_connections[kernel_id] -= 1 def kernel_model(self, kernel_id): """Return a JSON-safe dict representing a kernel For use in representing kernels in the JSON APIs. """ self._check_kernel_id(kernel_id) kernel = self._kernels[kernel_id] model = { "id": kernel_id, "name": kernel.kernel_name, "last_activity": isoformat(kernel.last_activity), "execution_state": kernel.execution_state, "connections": self._kernel_connections.get(kernel_id, 0), } return model def list_kernels(self): """Returns a list of kernel_id's of kernels running.""" kernels = [] kernel_ids = self.pinned_superclass.list_kernel_ids(self) for kernel_id in kernel_ids: try: model = self.kernel_model(kernel_id) kernels.append(model) except (web.HTTPError, KeyError): pass # Probably due to a (now) non-existent kernel, continue building the list return kernels # override _check_kernel_id to raise 404 instead of KeyError def _check_kernel_id(self, kernel_id): """Check a that a kernel_id exists and raise 404 if not.""" if kernel_id not in self: raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id) # monitoring activity: def start_watching_activity(self, kernel_id): """Start watching IOPub messages on a kernel for activity. - update last_activity on every message - record execution_state from status messages """ kernel = self._kernels[kernel_id] # add busy/activity markers: kernel.execution_state = 'starting' kernel.last_activity = utcnow() kernel._activity_stream = kernel.connect_iopub() session = Session( config=kernel.session.config, key=kernel.session.key, ) def record_activity(msg_list): """Record an IOPub message arriving from a kernel""" self.last_kernel_activity = kernel.last_activity = utcnow() idents, fed_msg_list = session.feed_identities(msg_list) msg = session.deserialize(fed_msg_list) msg_type = msg['header']['msg_type'] if msg_type == 'status': kernel.execution_state = msg['content']['execution_state'] self.log.debug("activity on %s: %s (%s)", kernel_id, msg_type, kernel.execution_state) else: self.log.debug("activity on %s: %s", kernel_id, msg_type) kernel._activity_stream.on_recv(record_activity) def stop_watching_activity(self, kernel_id): """Stop watching IOPub messages on a kernel for activity.""" kernel = self._kernels[kernel_id] if kernel._activity_stream: kernel._activity_stream.close() kernel._activity_stream = None def initialize_culler(self): """Start idle culler if 'cull_idle_timeout' is greater than zero. Regardless of that value, set flag that we've been here. """ if not self._initialized_culler and self.cull_idle_timeout > 0: if self._culler_callback is None: loop = IOLoop.current() if self.cull_interval <= 0: #handle case where user set invalid value self.log.warning( "Invalid value for 'cull_interval' detected (%s) - using default value (%s).", self.cull_interval, self.cull_interval_default) self.cull_interval = self.cull_interval_default self._culler_callback = PeriodicCallback( self.cull_kernels, 1000 * self.cull_interval) self.log.info( "Culling kernels with idle durations > %s seconds at %s second intervals ...", self.cull_idle_timeout, self.cull_interval) if self.cull_busy: self.log.info("Culling kernels even if busy") if self.cull_connected: self.log.info( "Culling kernels even with connected clients") self._culler_callback.start() self._initialized_culler = True async def cull_kernels(self): self.log.debug( "Polling every %s seconds for kernels idle > %s seconds...", self.cull_interval, self.cull_idle_timeout) """Create a separate list of kernels to avoid conflicting updates while iterating""" for kernel_id in list(self._kernels): try: await self.cull_kernel_if_idle(kernel_id) except Exception as e: self.log.exception( "The following exception was encountered while checking the idle duration of kernel %s: %s", kernel_id, e) async def cull_kernel_if_idle(self, kernel_id): kernel = self._kernels[kernel_id] if hasattr( kernel, 'last_activity' ): # last_activity is monkey-patched, so ensure that has occurred self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s", kernel_id, kernel.kernel_name, kernel.last_activity) dt_now = utcnow() dt_idle = dt_now - kernel.last_activity # Compute idle properties is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout) is_idle_execute = self.cull_busy or (kernel.execution_state != 'busy') connections = self._kernel_connections.get(kernel_id, 0) is_idle_connected = self.cull_connected or not connections # Cull the kernel if all three criteria are met if (is_idle_time and is_idle_execute and is_idle_connected): idle_duration = int(dt_idle.total_seconds()) self.log.warning( "Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.", kernel.execution_state, kernel.kernel_name, kernel_id, connections, idle_duration) await ensure_async(self.shutdown_kernel(kernel_id))
class InProcessKernel(IPythonKernel): #------------------------------------------------------------------------- # InProcessKernel interface #------------------------------------------------------------------------- # The frontends connected to this kernel. frontends = List( Instance('ipykernel.inprocess.client.InProcessKernelClient', allow_none=True)) # The GUI environment that the kernel is running under. This need not be # specified for the normal operation for the kernel, but is required for # IPython's GUI support (including pylab). The default is 'inline' because # it is safe under all GUI toolkits. gui = Enum(('tk', 'gtk', 'wx', 'qt', 'qt4', 'inline'), default_value='inline') raw_input_str = Any() stdout = Any() stderr = Any() #------------------------------------------------------------------------- # Kernel interface #------------------------------------------------------------------------- shell_class = Type(allow_none=True) shell_streams = List() control_stream = Any() iopub_socket = Instance(DummySocket, ()) stdin_socket = Instance(DummySocket, ()) def __init__(self, **traits): super(InProcessKernel, self).__init__(**traits) self.iopub_socket.on_trait_change(self._io_dispatch, 'message_sent') self.shell.kernel = self def execute_request(self, stream, ident, parent): """ Override for temporary IO redirection. """ with self._redirected_io(): super(InProcessKernel, self).execute_request(stream, ident, parent) def start(self): """ Override registration of dispatchers for streams. """ self.shell.exit_now = False def _abort_queue(self, stream): """ The in-process kernel doesn't abort requests. """ pass def _input_request(self, prompt, ident, parent, password=False): # Flush output before making the request. self.raw_input_str = None sys.stderr.flush() sys.stdout.flush() # Send the input request. content = json_clean(dict(prompt=prompt, password=password)) msg = self.session.msg(u'input_request', content, parent) for frontend in self.frontends: if frontend.session.session == parent['header']['session']: frontend.stdin_channel.call_handlers(msg) break else: logging.error('No frontend found for raw_input request') return str() # Await a response. while self.raw_input_str is None: frontend.stdin_channel.process_events() return self.raw_input_str #------------------------------------------------------------------------- # Protected interface #------------------------------------------------------------------------- @contextmanager def _redirected_io(self): """ Temporarily redirect IO to the kernel. """ sys_stdout, sys_stderr = sys.stdout, sys.stderr sys.stdout, sys.stderr = self.stdout, self.stderr yield sys.stdout, sys.stderr = sys_stdout, sys_stderr #------ Trait change handlers -------------------------------------------- def _io_dispatch(self): """ Called when a message is sent to the IO socket. """ ident, msg = self.session.recv(self.iopub_socket, copy=False) for frontend in self.frontends: frontend.iopub_channel.call_handlers(msg) #------ Trait initializers ----------------------------------------------- def _log_default(self): return logging.getLogger(__name__) def _session_default(self): from jupyter_client.session import Session return Session(parent=self, key=b'') def _shell_class_default(self): return InProcessInteractiveShell def _stdout_default(self): from ipykernel.iostream import OutStream return OutStream(self.session, self.iopub_socket, u'stdout', pipe=False) def _stderr_default(self): from ipykernel.iostream import OutStream return OutStream(self.session, self.iopub_socket, u'stderr', pipe=False)
class MWOAuthenticator(OAuthenticator): login_service = 'MediaWiki' login_handler = MWLoginHandler callback_handler = MWCallbackHandler mw_index_url = Unicode( os.environ.get('MW_INDEX_URL', 'https://meta.wikimedia.org/w/index.php'), config=True, help='Full path to index.php of the MW instance to use to log in' ) executor_threads = Integer(12, help="""Number of executor threads. MediaWiki OAuth requests happen in this thread, so it is mostly waiting for network replies. """, config=True, ) executor = Any() def normalize_username(self, username): """ Override normalize_username to avoid lowercasing usernames """ return username def _executor_default(self): return ThreadPoolExecutor(self.executor_threads) @gen.coroutine def authenticate(self, handler, data=None): consumer_token = ConsumerToken( self.client_id, self.client_secret, ) handshaker = Handshaker( self.mw_index_url, consumer_token ) request_token = dejsonify(handler.get_secure_cookie(AUTH_REQUEST_COOKIE_NAME)) handler.clear_cookie(AUTH_REQUEST_COOKIE_NAME) access_token = yield self.executor.submit( handshaker.complete, request_token, handler.request.query ) identity = yield self.executor.submit(handshaker.identify, access_token) if identity and 'username' in identity: # this shouldn't be necessary anymore, # but keep for backward-compatibility return { 'name': identity['username'].replace(' ', '_'), 'auth_state': { 'ACCESS_TOKEN_KEY': access_token.key.decode('utf-8'), 'ACCESS_TOKEN_SECRET': access_token.secret.decode('utf-8'), 'MEDIAWIKI_USER_IDENTITY': identity, } } else: self.log.error("No username found in %s", identity)
class ZMQTerminalInteractiveShell(SingletonConfigurable): readline_use = False pt_cli = None _executing = False _execution_state = Unicode('') _pending_clearoutput = False _eventloop = None editing_mode = Unicode( 'emacs', config=True, help="Shortcut style to use at the prompt. 'vi' or 'emacs'.", ) highlighting_style = Unicode( '', config=True, help="The name of a Pygments style to use for syntax highlighting") highlighting_style_overrides = Dict( config=True, help="Override highlighting format for specific tokens") true_color = Bool( False, config=True, help=("Use 24bit colors instead of 256 colors in prompt highlighting. " "If your terminal supports true color, the following command " "should print 'TRUECOLOR' in orange: " "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")) history_load_length = Integer( 1000, config=True, help="How many history items to load into memory") banner = Unicode( 'Jupyter console {version}\n\n{kernel_banner}', config=True, help=( "Text to display before the first prompt. Will be formatted with " "variables {version} and {kernel_banner}.")) kernel_timeout = Float( 60, config=True, help="""Timeout for giving up on a kernel (in seconds). On first connect and restart, the console tests whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take before being presumed dead. """) image_handler = Enum(('PIL', 'stream', 'tempfile', 'callable'), 'PIL', config=True, allow_none=True, help=""" Handler for image type output. This is useful, for example, when connecting to the kernel in which pylab inline backend is activated. There are four handlers defined. 'PIL': Use Python Imaging Library to popup image; 'stream': Use an external program to show the image. Image will be fed into the STDIN of the program. You will need to configure `stream_image_handler`; 'tempfile': Use an external program to show the image. Image will be saved in a temporally file and the program is called with the temporally file. You will need to configure `tempfile_image_handler`; 'callable': You can set any Python callable which is called with the image data. You will need to configure `callable_image_handler`. """) stream_image_handler = List(config=True, help=""" Command to invoke an image viewer program when you are using 'stream' image handler. This option is a list of string where the first element is the command itself and reminders are the options for the command. Raw image data is given as STDIN to the program. """) tempfile_image_handler = List(config=True, help=""" Command to invoke an image viewer program when you are using 'tempfile' image handler. This option is a list of string where the first element is the command itself and reminders are the options for the command. You can use {file} and {format} in the string to represent the location of the generated image file and image format. """) callable_image_handler = Any(config=True, help=""" Callable object called via 'callable' image handler with one argument, `data`, which is `msg["content"]["data"]` where `msg` is the message from iopub channel. For exmaple, you can find base64 encoded PNG data as `data['image/png']`. If your function can't handle the data supplied, it should return `False` to indicate this. """) mime_preference = List( default_value=['image/png', 'image/jpeg', 'image/svg+xml'], config=True, help=""" Preferred object representation MIME type in order. First matched MIME type will be used. """) use_kernel_is_complete = Bool( True, config=True, help="""Whether to use the kernel's is_complete message handling. If False, then the frontend will use its own is_complete handler. """) kernel_is_complete_timeout = Float( 1, config=True, help="""Timeout (in seconds) for giving up on a kernel's is_complete response. If the kernel does not respond at any point within this time, the kernel will no longer be asked if code is complete, and the console will default to the built-in is_complete test. """) confirm_exit = Bool(True, config=True, help="""Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', to force a direct exit without any confirmation. """) manager = Instance('jupyter_client.KernelManager', allow_none=True) client = Instance('jupyter_client.KernelClient', allow_none=True) def _client_changed(self, name, old, new): self.session_id = new.session.session session_id = Unicode() def _banner1_default(self): return "Jupyter Console {version}\n".format(version=__version__) def __init__(self, **kwargs): # This is where traits with a config_key argument are updated # from the values on config. super(ZMQTerminalInteractiveShell, self).__init__(**kwargs) self.configurables = [self] self.init_history() self.init_completer() self.init_io() self.init_kernel_info() self.init_prompt_toolkit_cli() self.keep_running = True self.execution_count = 1 def init_completer(self): """Initialize the completion machinery. This creates completion machinery that can be used by client code, either interactively in-process (typically triggered by the readline library), programmatically (such as in test suites) or out-of-process (typically over the network by remote frontends). """ self.Completer = ZMQCompleter(self, self.client, config=self.config) def init_history(self): """Sets up the command history. """ self.history_manager = ZMQHistoryManager(client=self.client) self.configurables.append(self.history_manager) def get_prompt_tokens(self, cli): return [ (Token.Prompt, 'In ['), (Token.PromptNum, str(self.execution_count)), (Token.Prompt, ']: '), ] def get_continuation_tokens(self, cli, width): return [ (Token.Prompt, (' ' * (width - 2)) + ': '), ] def get_out_prompt_tokens(self): return [(Token.OutPrompt, 'Out['), (Token.OutPromptNum, str(self.execution_count)), (Token.OutPrompt, ']: ')] def print_out_prompt(self): self.pt_cli.print_tokens(self.get_out_prompt_tokens()) kernel_info = {} def init_kernel_info(self): """Wait for a kernel to be ready, and store kernel info""" timeout = self.kernel_timeout tic = time.time() self.client.hb_channel.unpause() msg_id = self.client.kernel_info() while True: try: reply = self.client.get_shell_msg(timeout=1) except Empty: if (time.time() - tic) > timeout: raise RuntimeError( "Kernel didn't respond to kernel_info_request") else: if reply['parent_header'].get('msg_id') == msg_id: self.kernel_info = reply['content'] return def show_banner(self): print( self.banner.format(version=__version__, kernel_banner=self.kernel_info.get( 'banner', ''))) def init_prompt_toolkit_cli(self): if 'JUPYTER_CONSOLE_TEST' in os.environ: # Simple restricted interface for tests so we can find prompts with # pexpect. Multi-line input not supported. def prompt(): return cast_unicode_py2( input('In [%d]: ' % self.execution_count)) self.prompt_for_code = prompt self.print_out_prompt = \ lambda: print('Out[%d]: ' % self.execution_count, end='') return kbmanager = KeyBindingManager.for_prompt() insert_mode = ViInsertMode() | EmacsInsertMode() # Ctrl+J == Enter, seemingly @kbmanager.registry.add_binding(Keys.ControlJ, filter=(HasFocus(DEFAULT_BUFFER) & ~HasSelection() & insert_mode)) def _(event): b = event.current_buffer d = b.document if not (d.on_last_line or d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()): b.newline() return more, indent = self.check_complete(d.text) if (not more) and b.accept_action.is_returnable: b.accept_action.validate_and_handle(event.cli, b) else: b.insert_text('\n' + indent) @kbmanager.registry.add_binding(Keys.ControlC, filter=HasFocus(DEFAULT_BUFFER)) def _(event): event.current_buffer.reset() # Pre-populate history from IPython's history database history = InMemoryHistory() last_cell = u"" for _, _, cell in self.history_manager.get_tail( self.history_load_length, include_latest=True): # Ignore blank lines and consecutive duplicates cell = cell.rstrip() if cell and (cell != last_cell): history.append(cell) style_overrides = { Token.Prompt: '#009900', Token.PromptNum: '#00ff00 bold', Token.OutPrompt: '#ff2200', Token.OutPromptNum: '#ff0000 bold', } if self.highlighting_style: style_cls = get_style_by_name(self.highlighting_style) else: style_cls = get_style_by_name('default') # The default theme needs to be visible on both a dark background # and a light background, because we can't tell what the terminal # looks like. These tweaks to the default theme help with that. style_overrides.update({ Token.Number: '#007700', Token.Operator: 'noinherit', Token.String: '#BB6622', Token.Name.Function: '#2080D0', Token.Name.Class: 'bold #2080D0', Token.Name.Namespace: 'bold #2080D0', }) style_overrides.update(self.highlighting_style_overrides) style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls, style_dict=style_overrides) editing_mode = getattr(EditingMode, self.editing_mode.upper()) langinfo = self.kernel_info.get('language_info', {}) lexer = langinfo.get('pygments_lexer', langinfo.get('name', 'text')) app = create_prompt_application( multiline=True, editing_mode=editing_mode, lexer=PygmentsLexer(get_pygments_lexer(lexer)), get_prompt_tokens=self.get_prompt_tokens, get_continuation_tokens=self.get_continuation_tokens, key_bindings_registry=kbmanager.registry, history=history, completer=JupyterPTCompleter(self.Completer), enable_history_search=True, style=style, ) self._eventloop = create_eventloop() self.pt_cli = CommandLineInterface( app, eventloop=self._eventloop, output=create_output(true_color=self.true_color), ) def prompt_for_code(self): document = self.pt_cli.run(pre_run=self.pre_prompt, reset_current_buffer=True) return document.text def init_io(self): if sys.platform not in {'win32', 'cli'}: return import colorama colorama.init() def check_complete(self, code): if self.use_kernel_is_complete: msg_id = self.client.is_complete(code) try: return self.handle_is_complete_reply( msg_id, timeout=self.kernel_is_complete_timeout) except SyntaxError: return False, "" else: lines = code.splitlines() if len(lines): more = (lines[-1] != "") return more, "" else: return False, "" def ask_exit(self): self.keep_running = False # This is set from payloads in handle_execute_reply next_input = None def pre_prompt(self): if self.next_input: b = self.pt_cli.application.buffer b.text = cast_unicode_py2(self.next_input) self.next_input = None # Move the cursor to the end b.cursor_position += b.document.get_end_of_document_position() def interact(self, display_banner=None): while self.keep_running: print('\n', end='') try: code = self.prompt_for_code() except EOFError: if (not self.confirm_exit) \ or ask_yes_no('Do you really want to exit ([y]/n)?','y','n'): self.ask_exit() else: if code: self.run_cell(code, store_history=True) def mainloop(self): self.keepkernel = False # An extra layer of protection in case someone mashing Ctrl-C breaks # out of our internal code. while True: try: self.interact() break except KeyboardInterrupt: print("\nKeyboardInterrupt escaped interact()\n") self._eventloop.close() if self.keepkernel and not self.own_kernel: print('keeping kernel alive') elif self.keepkernel and self.own_kernel: print("owning kernel, cannot keep it alive") self.client.shutdown() else: print("Shutting down kernel") self.client.shutdown() def run_cell(self, cell, store_history=True): """Run a complete IPython cell. Parameters ---------- cell : str The code (including IPython code such as %magic functions) to run. store_history : bool If True, the raw and translated cell will be stored in IPython's history. For user code calling back into IPython's machinery, this should be set to False. """ if (not cell) or cell.isspace(): # pressing enter flushes any pending display self.handle_iopub() return # flush stale replies, which could have been ignored, due to missed heartbeats while self.client.shell_channel.msg_ready(): self.client.shell_channel.get_msg() # execute takes 'hidden', which is the inverse of store_hist msg_id = self.client.execute(cell, not store_history) # first thing is wait for any side effects (output, stdin, etc.) self._executing = True self._execution_state = "busy" while self._execution_state != 'idle' and self.client.is_alive(): try: self.handle_input_request(msg_id, timeout=0.05) except Empty: # display intermediate print statements, etc. self.handle_iopub(msg_id) except ZMQError as e: # Carry on if polling was interrupted by a signal if e.errno != errno.EINTR: raise # after all of that is done, wait for the execute reply while self.client.is_alive(): try: self.handle_execute_reply(msg_id, timeout=0.05) except Empty: pass else: break self._executing = False #----------------- # message handlers #----------------- def handle_execute_reply(self, msg_id, timeout=None): msg = self.client.shell_channel.get_msg(block=False, timeout=timeout) if msg["parent_header"].get("msg_id", None) == msg_id: self.handle_iopub(msg_id) content = msg["content"] status = content['status'] if status == 'aborted': self.write('Aborted\n') return elif status == 'ok': # handle payloads for item in content.get("payload", []): source = item['source'] if source == 'page': page.page(item['data']['text/plain']) elif source == 'set_next_input': self.next_input = item['text'] elif source == 'ask_exit': self.keepkernel = item.get('keepkernel', False) self.ask_exit() elif status == 'error': pass self.execution_count = int(content["execution_count"] + 1) def handle_is_complete_reply(self, msg_id, timeout=None): """ Wait for a repsonse from the kernel, and return two values: more? - (boolean) should the frontend ask for more input indent - an indent string to prefix the input Overloaded methods may want to examine the comeplete source. Its is in the self._source_lines_buffered list. """ ## Get the is_complete response: msg = None try: msg = self.client.shell_channel.get_msg(block=True, timeout=timeout) except Empty: warn('The kernel did not respond to an is_complete_request. ' 'Setting `use_kernel_is_complete` to False.') self.use_kernel_is_complete = False return False, "" ## Handle response: if msg["parent_header"].get("msg_id", None) != msg_id: warn( 'The kernel did not respond properly to an is_complete_request: %s.' % str(msg)) return False, "" else: status = msg["content"].get("status", None) indent = msg["content"].get("indent", "") ## Return more? and indent string if status == "complete": return False, indent elif status == "incomplete": return True, indent elif status == "invalid": raise SyntaxError() elif status == "unknown": return False, indent else: warn('The kernel sent an invalid is_complete_reply status: "%s".' % status) return False, indent include_other_output = Bool(False, config=True, help="""Whether to include output from clients other than this one sharing the same kernel. Outputs are not displayed until enter is pressed. """) other_output_prefix = Unicode( "[remote] ", config=True, help="""Prefix to add to outputs coming from clients other than this one. Only relevant if include_other_output is True. """) def from_here(self, msg): """Return whether a message is from this session""" return msg['parent_header'].get("session", self.session_id) == self.session_id def include_output(self, msg): """Return whether we should include a given output message""" from_here = self.from_here(msg) if msg['msg_type'] == 'execute_input': # only echo inputs not from here return self.include_other_output and not from_here if self.include_other_output: return True else: return from_here def handle_iopub(self, msg_id=''): """Process messages on the IOPub channel This method consumes and processes messages on the IOPub channel, such as stdout, stderr, execute_result and status. It only displays output that is caused by this session. """ while self.client.iopub_channel.msg_ready(): sub_msg = self.client.iopub_channel.get_msg() msg_type = sub_msg['header']['msg_type'] parent = sub_msg["parent_header"] if self.include_output(sub_msg): if msg_type == 'status': self._execution_state = sub_msg["content"][ "execution_state"] elif msg_type == 'stream': if sub_msg["content"]["name"] == "stdout": if self._pending_clearoutput: print("\r", end="") self._pending_clearoutput = False print(sub_msg["content"]["text"], end="") sys.stdout.flush() elif sub_msg["content"]["name"] == "stderr": if self._pending_clearoutput: print("\r", file=sys.stderr, end="") self._pending_clearoutput = False print(sub_msg["content"]["text"], file=sys.stderr, end="") sys.stderr.flush() elif msg_type == 'execute_result': if self._pending_clearoutput: print("\r", end="") self._pending_clearoutput = False self.execution_count = int( sub_msg["content"]["execution_count"]) if not self.from_here(sub_msg): sys.stdout.write(self.other_output_prefix) format_dict = sub_msg["content"]["data"] self.handle_rich_data(format_dict) if 'text/plain' not in format_dict: continue self.print_out_prompt() text_repr = format_dict['text/plain'] if '\n' in text_repr: # For multi-line results, start a new line after prompt print() print(text_repr) elif msg_type == 'display_data': data = sub_msg["content"]["data"] handled = self.handle_rich_data(data) if not handled: if not self.from_here(sub_msg): sys.stdout.write(self.other_output_prefix) # if it was an image, we handled it by now if 'text/plain' in data: print(data['text/plain']) elif msg_type == 'execute_input': content = sub_msg['content'] if not self.from_here(sub_msg): sys.stdout.write(self.other_output_prefix) sys.stdout.write('In [{}]: '.format( content['execution_count'])) sys.stdout.write(content['code']) elif msg_type == 'clear_output': if sub_msg["content"]["wait"]: self._pending_clearoutput = True else: print("\r", end="") elif msg_type == 'error': for frame in sub_msg["content"]["traceback"]: print(frame, file=sys.stderr) _imagemime = { 'image/png': 'png', 'image/jpeg': 'jpeg', 'image/svg+xml': 'svg', } def handle_rich_data(self, data): for mime in self.mime_preference: if mime in data and mime in self._imagemime: if self.handle_image(data, mime): return True return False def handle_image(self, data, mime): handler = getattr(self, 'handle_image_{0}'.format(self.image_handler), None) if handler: return handler(data, mime) def handle_image_PIL(self, data, mime): if mime not in ('image/png', 'image/jpeg'): return False try: from PIL import Image, ImageShow except ImportError: return False raw = base64.decodestring(data[mime].encode('ascii')) img = Image.open(BytesIO(raw)) return ImageShow.show(img) def handle_image_stream(self, data, mime): raw = base64.decodestring(data[mime].encode('ascii')) imageformat = self._imagemime[mime] fmt = dict(format=imageformat) args = [s.format(**fmt) for s in self.stream_image_handler] with open(os.devnull, 'w') as devnull: proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=devnull, stderr=devnull) proc.communicate(raw) return (proc.returncode == 0) def handle_image_tempfile(self, data, mime): raw = base64.decodestring(data[mime].encode('ascii')) imageformat = self._imagemime[mime] filename = 'tmp.{0}'.format(imageformat) with NamedFileInTemporaryDirectory(filename) as f, \ open(os.devnull, 'w') as devnull: f.write(raw) f.flush() fmt = dict(file=f.name, format=imageformat) args = [s.format(**fmt) for s in self.tempfile_image_handler] rc = subprocess.call(args, stdout=devnull, stderr=devnull) return (rc == 0) def handle_image_callable(self, data, mime): res = self.callable_image_handler(data) if res is not False: # If handler func returns e.g. None, assume it has handled the data. res = True return res def handle_input_request(self, msg_id, timeout=0.1): """ Method to capture raw_input """ req = self.client.stdin_channel.get_msg(timeout=timeout) # in case any iopub came while we were waiting: self.handle_iopub(msg_id) if msg_id == req["parent_header"].get("msg_id"): # wrap SIGINT handler real_handler = signal.getsignal(signal.SIGINT) def double_int(sig, frame): # call real handler (forwards sigint to kernel), # then raise local interrupt, stopping local raw_input real_handler(sig, frame) raise KeyboardInterrupt signal.signal(signal.SIGINT, double_int) content = req['content'] read = getpass if content.get('password', False) else input try: raw_data = read(content["prompt"]) except EOFError: # turn EOFError into EOF character raw_data = '\x04' except KeyboardInterrupt: sys.stdout.write('\n') return finally: # restore SIGINT handler signal.signal(signal.SIGINT, real_handler) # only send stdin reply if there *was not* another request # or execution finished while we were reading. if not (self.client.stdin_channel.msg_ready() or self.client.shell_channel.msg_ready()): self.client.input(raw_data)
class JupyterHub(Application): """An Application for starting a Multi-User Jupyter Notebook server.""" name = 'jupyterhub' version = jupyterhub.__version__ description = """Start a multi-user Jupyter Notebook server Spawns a configurable-http-proxy and multi-user Hub, which authenticates users and spawns single-user Notebook servers on behalf of users. """ examples = """ generate default config file: jupyterhub --generate-config -f /etc/jupyterhub/jupyterhub.py spawn the server on 10.0.1.2:443 with https: jupyterhub --ip 10.0.1.2 --port 443 --ssl-key my_ssl.key --ssl-cert my_ssl.cert """ aliases = Dict(aliases) flags = Dict(flags) subcommands = {'token': (NewToken, "Generate an API token for a user")} classes = List([ Spawner, LocalProcessSpawner, Authenticator, PAMAuthenticator, ]) config_file = Unicode( 'jupyterhub_config.py', config=True, help="The config file to load", ) generate_config = Bool( False, config=True, help="Generate default config file", ) answer_yes = Bool( False, config=True, help="Answer yes to any questions (e.g. confirm overwrite)") pid_file = Unicode('', config=True, help="""File to write PID Useful for daemonizing jupyterhub. """) cookie_max_age_days = Float( 14, config=True, help="""Number of days for a login cookie to be valid. Default is two weeks. """) last_activity_interval = Integer( 300, config=True, help= "Interval (in seconds) at which to update last-activity timestamps.") proxy_check_interval = Integer( 30, config=True, help="Interval (in seconds) at which to check if the proxy is running." ) data_files_path = Unicode( DATA_FILES_PATH, config=True, help= "The location of jupyterhub data files (e.g. /usr/local/share/jupyter/hub)" ) template_paths = List( config=True, help="Paths to search for jinja templates.", ) def _template_paths_default(self): return [os.path.join(self.data_files_path, 'templates')] ssl_key = Unicode( '', config=True, help="""Path to SSL key file for the public facing interface of the proxy Use with ssl_cert """) ssl_cert = Unicode( '', config=True, help= """Path to SSL certificate file for the public facing interface of the proxy Use with ssl_key """) ip = Unicode('', config=True, help="The public facing ip of the proxy") port = Integer(8000, config=True, help="The public facing port of the proxy") base_url = URLPrefix('/', config=True, help="The base URL of the entire application") jinja_environment_options = Dict( config=True, help="Supply extra arguments that will be passed to Jinja environment." ) proxy_cmd = Command('configurable-http-proxy', config=True, help="""The command to start the http proxy. Only override if configurable-http-proxy is not on your PATH """) debug_proxy = Bool(False, config=True, help="show debug output in configurable-http-proxy") proxy_auth_token = Unicode(config=True, help="""The Proxy Auth token. Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default. """) def _proxy_auth_token_default(self): token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None) if not token: self.log.warn('\n'.join([ "", "Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.", "Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.", "", ])) token = orm.new_token() return token proxy_api_ip = Unicode('localhost', config=True, help="The ip for the proxy API handlers") proxy_api_port = Integer(config=True, help="The port for the proxy API handlers") def _proxy_api_port_default(self): return self.port + 1 hub_port = Integer(8081, config=True, help="The port for this process") hub_ip = Unicode('localhost', config=True, help="The ip for this process") hub_prefix = URLPrefix( '/hub/', config=True, help="The prefix for the hub server. Must not be '/'") def _hub_prefix_default(self): return url_path_join(self.base_url, '/hub/') def _hub_prefix_changed(self, name, old, new): if new == '/': raise TraitError("'/' is not a valid hub prefix") if not new.startswith(self.base_url): self.hub_prefix = url_path_join(self.base_url, new) cookie_secret = Bytes(config=True, env='JPY_COOKIE_SECRET', help="""The cookie secret to use to encrypt cookies. Loaded from the JPY_COOKIE_SECRET env variable by default. """) cookie_secret_file = Unicode( 'jupyterhub_cookie_secret', config=True, help="""File in which to store the cookie secret.""") authenticator_class = Type(PAMAuthenticator, Authenticator, config=True, help="""Class for authenticating users. This should be a class with the following form: - constructor takes one kwarg: `config`, the IPython config object. - is a tornado.gen.coroutine - returns username on success, None on failure - takes two arguments: (handler, data), where `handler` is the calling web.RequestHandler, and `data` is the POST form data from the login page. """) authenticator = Instance(Authenticator) def _authenticator_default(self): return self.authenticator_class(parent=self, db=self.db) # class for spawning single-user servers spawner_class = Type( LocalProcessSpawner, Spawner, config=True, help="""The class to use for spawning single-user servers. Should be a subclass of Spawner. """) db_url = Unicode( 'sqlite:///jupyterhub.sqlite', config=True, help="url for the database. e.g. `sqlite:///jupyterhub.sqlite`") def _db_url_changed(self, name, old, new): if '://' not in new: # assume sqlite, if given as a plain filename self.db_url = 'sqlite:///%s' % new db_kwargs = Dict( config=True, help="""Include any kwargs to pass to the database connection. See sqlalchemy.create_engine for details. """) reset_db = Bool(False, config=True, help="Purge and reset the database.") debug_db = Bool( False, config=True, help="log all database transactions. This has A LOT of output") db = Any() session_factory = Any() admin_access = Bool( False, config=True, help="""Grant admin users permission to access single-user servers. Users should be properly informed if this is enabled. """) admin_users = Set( config=True, help="""DEPRECATED, use Authenticator.admin_users instead.""") tornado_settings = Dict(config=True) cleanup_servers = Bool( True, config=True, help="""Whether to shutdown single-user servers when the Hub shuts down. Disable if you want to be able to teardown the Hub while leaving the single-user servers running. If both this and cleanup_proxy are False, sending SIGINT to the Hub will only shutdown the Hub, leaving everything else running. The Hub should be able to resume from database state. """) cleanup_proxy = Bool( True, config=True, help="""Whether to shutdown the proxy when the Hub shuts down. Disable if you want to be able to teardown the Hub while leaving the proxy running. Only valid if the proxy was starting by the Hub process. If both this and cleanup_servers are False, sending SIGINT to the Hub will only shutdown the Hub, leaving everything else running. The Hub should be able to resume from database state. """) handlers = List() _log_formatter_cls = CoroutineLogFormatter http_server = None proxy_process = None io_loop = None def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%Y-%m-%d %H:%M:%S" def _log_format_default(self): """override default log format to include time""" return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s" extra_log_file = Unicode("", config=True, help="Set a logging.FileHandler on this file.") extra_log_handlers = List( Instance(logging.Handler), config=True, help="Extra log handlers to set on JupyterHub logger", ) def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False if self.extra_log_file: self.extra_log_handlers.append( logging.FileHandler(self.extra_log_file)) _formatter = self._log_formatter_cls( fmt=self.log_format, datefmt=self.log_datefmt, ) for handler in self.extra_log_handlers: if handler.formatter is None: handler.setFormatter(_formatter) self.log.addHandler(handler) # hook up tornado 3's loggers to our app handlers for log in (app_log, access_log, gen_log): # ensure all log statements identify the application they come from log.name = self.log.name logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_ports(self): if self.hub_port == self.port: raise TraitError( "The hub and proxy cannot both listen on port %i" % self.port) if self.hub_port == self.proxy_api_port: raise TraitError( "The hub and proxy API cannot both listen on port %i" % self.hub_port) if self.proxy_api_port == self.port: raise TraitError( "The proxy's public and API ports cannot both be %i" % self.port) @staticmethod def add_url_prefix(prefix, handlers): """add a url prefix to handlers""" for i, tup in enumerate(handlers): lis = list(tup) lis[0] = url_path_join(prefix, tup[0]) handlers[i] = tuple(lis) return handlers def init_handlers(self): h = [] h.extend(handlers.default_handlers) h.extend(apihandlers.default_handlers) # load handlers from the authenticator h.extend(self.authenticator.get_handlers(self)) self.handlers = self.add_url_prefix(self.hub_prefix, h) # some extra handlers, outside hub_prefix self.handlers.extend([ (r"%s" % self.hub_prefix.rstrip('/'), web.RedirectHandler, { "url": self.hub_prefix, "permanent": False, }), (r"(?!%s).*" % self.hub_prefix, handlers.PrefixRedirectHandler), (r'(.*)', handlers.Template404), ]) def _check_db_path(self, path): """More informative log messages for failed filesystem access""" path = os.path.abspath(path) parent, fname = os.path.split(path) user = getuser() if not os.path.isdir(parent): self.log.error("Directory %s does not exist", parent) if os.path.exists(parent) and not os.access(parent, os.W_OK): self.log.error("%s cannot create files in %s", user, parent) if os.path.exists(path) and not os.access(path, os.W_OK): self.log.error("%s cannot edit %s", user, path) def init_secrets(self): trait_name = 'cookie_secret' trait = self.traits()[trait_name] env_name = trait.get_metadata('env') secret_file = os.path.abspath( os.path.expanduser(self.cookie_secret_file)) secret = self.cookie_secret secret_from = 'config' # load priority: 1. config, 2. env, 3. file if not secret and os.environ.get(env_name): secret_from = 'env' self.log.info("Loading %s from env[%s]", trait_name, env_name) secret = binascii.a2b_hex(os.environ[env_name]) if not secret and os.path.exists(secret_file): secret_from = 'file' perm = os.stat(secret_file).st_mode if perm & 0o077: self.log.error("Bad permissions on %s", secret_file) else: self.log.info("Loading %s from %s", trait_name, secret_file) with open(secret_file) as f: b64_secret = f.read() try: secret = binascii.a2b_base64(b64_secret) except Exception as e: self.log.error("%s does not contain b64 key: %s", secret_file, e) if not secret: secret_from = 'new' self.log.debug("Generating new %s", trait_name) secret = os.urandom(SECRET_BYTES) if secret_file and secret_from == 'new': # if we generated a new secret, store it in the secret_file self.log.info("Writing %s to %s", trait_name, secret_file) b64_secret = binascii.b2a_base64(secret).decode('ascii') with open(secret_file, 'w') as f: f.write(b64_secret) try: os.chmod(secret_file, 0o600) except OSError: self.log.warn("Failed to set permissions on %s", secret_file) # store the loaded trait value self.cookie_secret = secret def init_db(self): """Create the database connection""" self.log.debug("Connecting to db: %s", self.db_url) try: self.session_factory = orm.new_session_factory(self.db_url, reset=self.reset_db, echo=self.debug_db, **self.db_kwargs) self.db = scoped_session(self.session_factory)() except OperationalError as e: self.log.error("Failed to connect to db: %s", self.db_url) self.log.debug("Database error was:", exc_info=True) if self.db_url.startswith('sqlite:///'): self._check_db_path(self.db_url.split(':///', 1)[1]) self.exit(1) def init_hub(self): """Load the Hub config into the database""" self.hub = self.db.query(orm.Hub).first() if self.hub is None: self.hub = orm.Hub(server=orm.Server( ip=self.hub_ip, port=self.hub_port, base_url=self.hub_prefix, cookie_name='jupyter-hub-token', )) self.db.add(self.hub) else: server = self.hub.server server.ip = self.hub_ip server.port = self.hub_port server.base_url = self.hub_prefix self.db.commit() @gen.coroutine def init_users(self): """Load users into and from the database""" db = self.db if self.admin_users and not self.authenticator.admin_users: self.log.warn("\nJupyterHub.admin_users is deprecated." "\nUse Authenticator.admin_users instead.") self.authenticator.admin_users = self.admin_users admin_users = self.authenticator.admin_users if not admin_users: # add current user as admin if there aren't any others admins = db.query(orm.User).filter(orm.User.admin == True) if admins.first() is None: admin_users.add(getuser()) new_users = [] for name in admin_users: # ensure anyone specified as admin in config is admin in db user = orm.User.find(db, name) if user is None: user = orm.User(name=name, admin=True) new_users.append(user) db.add(user) else: user.admin = True # the admin_users config variable will never be used after this point. # only the database values will be referenced. whitelist = self.authenticator.whitelist if not whitelist: self.log.info( "Not using whitelist. Any authenticated user will be allowed.") # add whitelisted users to the db for name in whitelist: user = orm.User.find(db, name) if user is None: user = orm.User(name=name) new_users.append(user) db.add(user) if whitelist: # fill the whitelist with any users loaded from the db, # so we are consistent in both directions. # This lets whitelist be used to set up initial list, # but changes to the whitelist can occur in the database, # and persist across sessions. for user in db.query(orm.User): whitelist.add(user.name) # The whitelist set and the users in the db are now the same. # From this point on, any user changes should be done simultaneously # to the whitelist set and user db, unless the whitelist is empty (all users allowed). db.commit() for user in new_users: yield gen.maybe_future(self.authenticator.add_user(user)) db.commit() @gen.coroutine def init_spawners(self): db = self.db user_summaries = [''] def _user_summary(user): parts = ['{0: >8}'.format(user.name)] if user.admin: parts.append('admin') if user.server: parts.append('running at %s' % user.server) return ' '.join(parts) @gen.coroutine def user_stopped(user): status = yield user.spawner.poll() self.log.warn( "User %s server stopped with exit code: %s", user.name, status, ) yield self.proxy.delete_user(user) yield user.stop() for user in db.query(orm.User): if not user.state: # without spawner state, server isn't valid user.server = None user_summaries.append(_user_summary(user)) continue self.log.debug("Loading state for %s from db", user.name) user.spawner = spawner = self.spawner_class( user=user, hub=self.hub, config=self.config, db=self.db, ) status = yield spawner.poll() if status is None: self.log.info("%s still running", user.name) spawner.add_poll_callback(user_stopped, user) spawner.start_polling() else: # user not running. This is expected if server is None, # but indicates the user's server died while the Hub wasn't running # if user.server is defined. log = self.log.warn if user.server else self.log.debug log("%s not running.", user.name) user.server = None user_summaries.append(_user_summary(user)) self.log.debug("Loaded users: %s", '\n'.join(user_summaries)) db.commit() def init_proxy(self): """Load the Proxy config into the database""" self.proxy = self.db.query(orm.Proxy).first() if self.proxy is None: self.proxy = orm.Proxy( public_server=orm.Server(), api_server=orm.Server(), ) self.db.add(self.proxy) self.db.commit() self.proxy.auth_token = self.proxy_auth_token # not persisted self.proxy.log = self.log self.proxy.public_server.ip = self.ip self.proxy.public_server.port = self.port self.proxy.api_server.ip = self.proxy_api_ip self.proxy.api_server.port = self.proxy_api_port self.proxy.api_server.base_url = '/api/routes/' self.db.commit() @gen.coroutine def start_proxy(self): """Actually start the configurable-http-proxy""" # check for proxy if self.proxy.public_server.is_up() or self.proxy.api_server.is_up(): # check for *authenticated* access to the proxy (auth token can change) try: yield self.proxy.get_routes() except (HTTPError, OSError, socket.error) as e: if isinstance(e, HTTPError) and e.code == 403: msg = "Did CONFIGPROXY_AUTH_TOKEN change?" else: msg = "Is something else using %s?" % self.proxy.public_server.bind_url self.log.error( "Proxy appears to be running at %s, but I can't access it (%s)\n%s", self.proxy.public_server.bind_url, e, msg) self.exit(1) return else: self.log.info("Proxy already running at: %s", self.proxy.public_server.bind_url) self.proxy_process = None return env = os.environ.copy() env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token cmd = self.proxy_cmd + [ '--ip', self.proxy.public_server.ip, '--port', str(self.proxy.public_server.port), '--api-ip', self.proxy.api_server.ip, '--api-port', str(self.proxy.api_server.port), '--default-target', self.hub.server.host, ] if self.debug_proxy: cmd.extend(['--log-level', 'debug']) if self.ssl_key: cmd.extend(['--ssl-key', self.ssl_key]) if self.ssl_cert: cmd.extend(['--ssl-cert', self.ssl_cert]) self.log.info("Starting proxy @ %s", self.proxy.public_server.bind_url) self.log.debug("Proxy cmd: %s", cmd) try: self.proxy_process = Popen(cmd, env=env) except FileNotFoundError as e: self.log.error( "Failed to find proxy %r\n" "The proxy can be installed with `npm install -g configurable-http-proxy`" % self.proxy_cmd) self.exit(1) def _check(): status = self.proxy_process.poll() if status is not None: e = RuntimeError("Proxy failed to start with exit code %i" % status) # py2-compatible `raise e from None` e.__cause__ = None raise e for server in (self.proxy.public_server, self.proxy.api_server): for i in range(10): _check() try: yield server.wait_up(1) except TimeoutError: continue else: break yield server.wait_up(1) self.log.debug("Proxy started and appears to be up") @gen.coroutine def check_proxy(self): if self.proxy_process.poll() is None: return self.log.error( "Proxy stopped with exit code %r", 'unknown' if self.proxy_process is None else self.proxy_process.poll()) yield self.start_proxy() self.log.info("Setting up routes on new proxy") yield self.proxy.add_all_users() self.log.info("New proxy back up, and good to go") def init_tornado_settings(self): """Set up the tornado settings dict.""" base_url = self.hub.server.base_url jinja_env = Environment(loader=FileSystemLoader(self.template_paths), **self.jinja_environment_options) login_url = self.authenticator.login_url(base_url) logout_url = self.authenticator.logout_url(base_url) # if running from git, disable caching of require.js # otherwise cache based on server start time parent = os.path.dirname(os.path.dirname(jupyterhub.__file__)) if os.path.isdir(os.path.join(parent, '.git')): version_hash = '' else: version_hash = datetime.now().strftime("%Y%m%d%H%M%S"), settings = dict( log_function=log_request, config=self.config, log=self.log, db=self.db, proxy=self.proxy, hub=self.hub, admin_users=self.authenticator.admin_users, admin_access=self.admin_access, authenticator=self.authenticator, spawner_class=self.spawner_class, base_url=self.base_url, cookie_secret=self.cookie_secret, cookie_max_age_days=self.cookie_max_age_days, login_url=login_url, logout_url=logout_url, static_path=os.path.join(self.data_files_path, 'static'), static_url_prefix=url_path_join(self.hub.server.base_url, 'static/'), static_handler_class=CacheControlStaticFilesHandler, template_path=self.template_paths, jinja2_env=jinja_env, version_hash=version_hash, ) # allow configured settings to have priority settings.update(self.tornado_settings) self.tornado_settings = settings def init_tornado_application(self): """Instantiate the tornado Application object""" self.tornado_application = web.Application(self.handlers, **self.tornado_settings) def write_pid_file(self): pid = os.getpid() if self.pid_file: self.log.debug("Writing PID %i to %s", pid, self.pid_file) with open(self.pid_file, 'w') as f: f.write('%i' % pid) @gen.coroutine @catch_config_error def initialize(self, *args, **kwargs): super().initialize(*args, **kwargs) if self.generate_config or self.subapp: return self.load_config_file(self.config_file) self.init_logging() if 'JupyterHubApp' in self.config: self.log.warn( "Use JupyterHub in config, not JupyterHubApp. Outdated config:\n%s", '\n'.join('JupyterHubApp.{key} = {value!r}'.format(key=key, value=value) for key, value in self.config.JupyterHubApp.items())) cfg = self.config.copy() cfg.JupyterHub.merge(cfg.JupyterHubApp) self.update_config(cfg) self.write_pid_file() self.init_ports() self.init_secrets() self.init_db() self.init_hub() self.init_proxy() yield self.init_users() yield self.init_spawners() self.init_handlers() self.init_tornado_settings() self.init_tornado_application() @gen.coroutine def cleanup(self): """Shutdown our various subprocesses and cleanup runtime files.""" futures = [] if self.cleanup_servers: self.log.info("Cleaning up single-user servers...") # request (async) process termination for user in self.db.query(orm.User): if user.spawner is not None: futures.append(user.stop()) else: self.log.info("Leaving single-user servers running") # clean up proxy while SUS are shutting down if self.cleanup_proxy: if self.proxy_process: self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid) if self.proxy_process.poll() is None: try: self.proxy_process.terminate() except Exception as e: self.log.error("Failed to terminate proxy process: %s", e) else: self.log.info("I didn't start the proxy, I can't clean it up") else: self.log.info("Leaving proxy running") # wait for the requests to stop finish: for f in futures: try: yield f except Exception as e: self.log.error("Failed to stop user: %s", e) self.db.commit() if self.pid_file and os.path.exists(self.pid_file): self.log.info("Cleaning up PID file %s", self.pid_file) os.remove(self.pid_file) # finally stop the loop once we are all cleaned up self.log.info("...done") def write_config_file(self): """Write our default config to a .py config file""" if os.path.exists(self.config_file) and not self.answer_yes: answer = '' def ask(): prompt = "Overwrite %s with default config? [y/N]" % self.config_file try: return input(prompt).lower() or 'n' except KeyboardInterrupt: print('') # empty line return 'n' answer = ask() while not answer.startswith(('y', 'n')): print("Please answer 'yes' or 'no'") answer = ask() if answer.startswith('n'): return config_text = self.generate_config_file() if isinstance(config_text, bytes): config_text = config_text.decode('utf8') print("Writing default config to: %s" % self.config_file) with open(self.config_file, mode='w') as f: f.write(config_text) @gen.coroutine def update_last_activity(self): """Update User.last_activity timestamps from the proxy""" routes = yield self.proxy.get_routes() for prefix, route in routes.items(): if 'user' not in route: # not a user route, ignore it continue user = orm.User.find(self.db, route['user']) if user is None: self.log.warn("Found no user for route: %s", route) continue try: dt = datetime.strptime(route['last_activity'], ISO8601_ms) except Exception: dt = datetime.strptime(route['last_activity'], ISO8601_s) user.last_activity = max(user.last_activity, dt) self.db.commit() yield self.proxy.check_routes(routes) @gen.coroutine def start(self): """Start the whole thing""" self.io_loop = loop = IOLoop.current() if self.subapp: self.subapp.start() loop.stop() return if self.generate_config: self.write_config_file() loop.stop() return # start the webserver self.http_server = tornado.httpserver.HTTPServer( self.tornado_application, xheaders=True) try: self.http_server.listen(self.hub_port, address=self.hub_ip) except Exception: self.log.error("Failed to bind hub to %s", self.hub.server.bind_url) raise else: self.log.info("Hub API listening on %s", self.hub.server.bind_url) # start the proxy try: yield self.start_proxy() except Exception as e: self.log.critical("Failed to start proxy", exc_info=True) self.exit(1) return loop.add_callback(self.proxy.add_all_users) if self.proxy_process: # only check / restart the proxy if we started it in the first place. # this means a restarted Hub cannot restart a Proxy that its # predecessor started. pc = PeriodicCallback(self.check_proxy, 1e3 * self.proxy_check_interval) pc.start() if self.last_activity_interval: pc = PeriodicCallback(self.update_last_activity, 1e3 * self.last_activity_interval) pc.start() self.log.info("JupyterHub is now running at %s", self.proxy.public_server.url) # register cleanup on both TERM and INT atexit.register(self.atexit) self.init_signal() def init_signal(self): signal.signal(signal.SIGTERM, self.sigterm) def sigterm(self, signum, frame): self.log.critical("Received SIGTERM, shutting down") self.io_loop.stop() self.atexit() _atexit_ran = False def atexit(self): """atexit callback""" if self._atexit_ran: return self._atexit_ran = True # run the cleanup step (in a new loop, because the interrupted one is unclean) IOLoop.clear_current() loop = IOLoop() loop.make_current() loop.run_sync(self.cleanup) def stop(self): if not self.io_loop: return if self.http_server: if self.io_loop._running: self.io_loop.add_callback(self.http_server.stop) else: self.http_server.stop() self.io_loop.add_callback(self.io_loop.stop) @gen.coroutine def launch_instance_async(self, argv=None): try: yield self.initialize(argv) yield self.start() except Exception as e: self.log.exception("") self.exit(1) @classmethod def launch_instance(cls, argv=None): self = cls.instance() loop = IOLoop.current() loop.add_callback(self.launch_instance_async, argv) try: loop.start() except KeyboardInterrupt: print("\nInterrupted")
class FileContentsManager(FileManagerMixin, ContentsManager): root_dir = Unicode(config=True) @default('root_dir') def _default_root_dir(self): try: return self.parent.notebook_dir except AttributeError: return getcwd() save_script = Bool( False, config=True, help='DEPRECATED, use post_save_hook. Will be removed in Notebook 5.0') @observe('save_script') def _update_save_script(self, change): if not change['new']: return self.log.warning(""" `--script` is deprecated and will be removed in notebook 5.0. You can trigger nbconvert via pre- or post-save hooks: ContentsManager.pre_save_hook FileContentsManager.post_save_hook A post-save hook has been registered that calls: jupyter nbconvert --to script [notebook] which behaves similarly to `--script`. """) self.post_save_hook = _post_save_script post_save_hook = Any(None, config=True, allow_none=True, help="""Python callable or importstring thereof to be called on the path of a file just saved. This can be used to process the file on disk, such as converting the notebook to a script or HTML via nbconvert. It will be called as (all arguments passed by keyword):: hook(os_path=os_path, model=model, contents_manager=instance) - path: the filesystem path to the file just written - model: the model representing the file - contents_manager: this ContentsManager instance """) @validate('post_save_hook') def _validate_post_save_hook(self, proposal): value = proposal['value'] if isinstance(value, string_types): value = import_item(value) if not callable(value): raise TraitError("post_save_hook must be callable") return value def run_post_save_hook(self, model, os_path): """Run the post-save hook if defined, and log errors""" if self.post_save_hook: try: self.log.debug("Running post-save hook on %s", os_path) self.post_save_hook(os_path=os_path, model=model, contents_manager=self) except Exception as e: self.log.error("Post-save hook failed o-n %s", os_path, exc_info=True) raise web.HTTPError( 500, u'Unexpected error while running post hook save: %s' % e) @validate('root_dir') def _validate_root_dir(self, proposal): """Do a bit of validation of the root_dir.""" value = proposal['value'] if not os.path.isabs(value): # If we receive a non-absolute path, make it absolute. value = os.path.abspath(value) if not os.path.isdir(value): raise TraitError("%r is not a directory" % value) return value @default('checkpoints_class') def _checkpoints_class_default(self): return FileCheckpoints delete_to_trash = Bool( True, config=True, help="""If True (default), deleting files will send them to the platform's trash/recycle bin, where they can be recovered. If False, deleting files really deletes them.""") @default('files_handler_class') def _files_handler_class_default(self): return AuthenticatedFileHandler @default('files_handler_params') def _files_handler_params_default(self): return {'path': self.root_dir} def is_hidden(self, path): """Does the API style path correspond to a hidden directory or file? Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to root_dir). Returns ------- hidden : bool Whether the path exists and is hidden. """ path = path.strip('/') os_path = self._get_os_path(path=path) return is_hidden(os_path, self.root_dir) def file_exists(self, path): """Returns True if the file exists, else returns False. API-style wrapper for os.path.isfile Parameters ---------- path : string The relative path to the file (with '/' as separator) Returns ------- exists : bool Whether the file exists. """ path = path.strip('/') os_path = self._get_os_path(path) return os.path.isfile(os_path) def dir_exists(self, path): """Does the API-style path refer to an extant directory? API-style wrapper for os.path.isdir Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to root_dir). Returns ------- exists : bool Whether the path is indeed a directory. """ path = path.strip('/') os_path = self._get_os_path(path=path) return os.path.isdir(os_path) def exists(self, path): """Returns True if the path exists, else returns False. API-style wrapper for os.path.exists Parameters ---------- path : string The API path to the file (with '/' as separator) Returns ------- exists : bool Whether the target exists. """ path = path.strip('/') os_path = self._get_os_path(path=path) return exists(os_path) def _base_model(self, path): """Build the common base of a contents model""" os_path = self._get_os_path(path) info = os.lstat(os_path) try: last_modified = tz.utcfromtimestamp(info.st_mtime) except (ValueError, OSError): # Files can rarely have an invalid timestamp # https://github.com/jupyter/notebook/issues/2539 # https://github.com/jupyter/notebook/issues/2757 # Use the Unix epoch as a fallback so we don't crash. self.log.warning('Invalid mtime %s for %s', info.st_mtime, os_path) last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC) try: created = tz.utcfromtimestamp(info.st_ctime) except (ValueError, OSError): # See above self.log.warning('Invalid ctime %s for %s', info.st_ctime, os_path) created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC) # Create the base model. model = {} model['name'] = path.rsplit('/', 1)[-1] model['path'] = path model['last_modified'] = last_modified model['created'] = created model['content'] = None model['format'] = None model['mimetype'] = None try: model['writable'] = os.access(os_path, os.W_OK) except OSError: self.log.error("Failed to check write permissions on %s", os_path) model['writable'] = False return model def _dir_model(self, path, content=True): """Build a model for a directory if content is requested, will include a listing of the directory """ os_path = self._get_os_path(path) four_o_four = u'directory does not exist: %r' % path if not os.path.isdir(os_path): raise web.HTTPError(404, four_o_four) elif is_hidden(os_path, self.root_dir) and not self.allow_hidden: self.log.info( "Refusing to serve hidden directory %r, via 404 Error", os_path) raise web.HTTPError(404, four_o_four) model = self._base_model(path) model['type'] = 'directory' if content: model['content'] = contents = [] os_dir = self._get_os_path(path) for name in os.listdir(os_dir): try: os_path = os.path.join(os_dir, name) except UnicodeDecodeError as e: self.log.warning("failed to decode filename '%s': %s", name, e) continue try: st = os.lstat(os_path) except OSError as e: # skip over broken symlinks in listing if e.errno == errno.ENOENT: self.log.warning("%s doesn't exist", os_path) else: self.log.warning("Error stat-ing %s: %s", os_path, e) continue if (not stat.S_ISLNK(st.st_mode) and not stat.S_ISREG(st.st_mode) and not stat.S_ISDIR(st.st_mode)): self.log.debug("%s not a regular file", os_path) continue if self.should_list(name) and not is_file_hidden(os_path, stat_res=st): contents.append( self.get(path='%s/%s' % (path, name), content=False)) model['format'] = 'json' return model def _file_model(self, path, content=True, format=None): """Build a model for a file if content is requested, include the file contents. format: If 'text', the contents will be decoded as UTF-8. If 'base64', the raw bytes contents will be encoded as base64. If not specified, try to decode as UTF-8, and fall back to base64 """ model = self._base_model(path) model['type'] = 'file' os_path = self._get_os_path(path) model['mimetype'] = mimetypes.guess_type(os_path)[0] if content: content, format = self._read_file(os_path, format) if model['mimetype'] is None: default_mime = { 'text': 'text/plain', 'base64': 'application/octet-stream' }[format] model['mimetype'] = default_mime model.update( content=content, format=format, ) return model def _notebook_model(self, path, content=True): """Build a notebook model if content is requested, the notebook content will be populated as a JSON structure (not double-serialized) """ model = self._base_model(path) model['type'] = 'notebook' if content: os_path = self._get_os_path(path) nb = self._read_notebook(os_path, as_version=4) self.mark_trusted_cells(nb, path) model['content'] = nb model['format'] = 'json' self.validate_notebook_model(model) return model def get(self, path, content=True, type=None, format=None): """ Takes a path for an entity and returns its model Parameters ---------- path : str the API path that describes the relative path for the target content : bool Whether to include the contents in the reply type : str, optional The requested type - 'file', 'notebook', or 'directory'. Will raise HTTPError 400 if the content doesn't match. format : str, optional The requested format for file contents. 'text' or 'base64'. Ignored if this returns a notebook or directory model. Returns ------- model : dict the contents model. If content=True, returns the contents of the file or directory as well. """ path = path.strip('/') if not self.exists(path): raise web.HTTPError(404, u'No such file or directory: %s' % path) os_path = self._get_os_path(path) if os.path.isdir(os_path): if type not in (None, 'directory'): raise web.HTTPError(400, u'%s is a directory, not a %s' % (path, type), reason='bad type') model = self._dir_model(path, content=content) elif type == 'notebook' or (type is None and path.endswith('.ipynb')): model = self._notebook_model(path, content=content) else: if type == 'directory': raise web.HTTPError(400, u'%s is not a directory' % path, reason='bad type') model = self._file_model(path, content=content, format=format) return model def _save_directory(self, os_path, model, path=''): """create a directory""" if is_hidden(os_path, self.root_dir) and not self.allow_hidden: raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path) if not os.path.exists(os_path): with self.perm_to_403(): os.mkdir(os_path) elif not os.path.isdir(os_path): raise web.HTTPError(400, u'Not a directory: %s' % (os_path)) else: self.log.debug("Directory %r already exists", os_path) def save(self, model, path=''): """Save the file model and return the model with no content.""" path = path.strip('/') if 'type' not in model: raise web.HTTPError(400, u'No file type provided') if 'content' not in model and model['type'] != 'directory': raise web.HTTPError(400, u'No file content provided') os_path = self._get_os_path(path) self.log.debug("Saving %s", os_path) self.run_pre_save_hook(model=model, path=path) try: if model['type'] == 'notebook': nb = nbformat.from_dict(model['content']) self.check_and_sign(nb, path) self._save_notebook(os_path, nb) # One checkpoint should always exist for notebooks. if not self.checkpoints.list_checkpoints(path): self.create_checkpoint(path) elif model['type'] == 'file': # Missing format will be handled internally by _save_file. self._save_file(os_path, model['content'], model.get('format')) elif model['type'] == 'directory': self._save_directory(os_path, model, path) else: raise web.HTTPError( 400, "Unhandled contents type: %s" % model['type']) except web.HTTPError: raise except Exception as e: self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True) raise web.HTTPError( 500, u'Unexpected error while saving file: %s %s' % (path, e)) validation_message = None if model['type'] == 'notebook': self.validate_notebook_model(model) validation_message = model.get('message', None) model = self.get(path, content=False) if validation_message: model['message'] = validation_message self.run_post_save_hook(model=model, os_path=os_path) return model def delete_file(self, path): """Delete file at path.""" path = path.strip('/') os_path = self._get_os_path(path) rm = os.unlink if not os.path.exists(os_path): raise web.HTTPError( 404, u'File or directory does not exist: %s' % os_path) if self.delete_to_trash: self.log.debug("Sending %s to trash", os_path) # Looking at the code in send2trash, I don't think the errors it # raises let us distinguish permission errors from other errors in # code. So for now, just let them all get logged as server errors. send2trash(os_path) return if os.path.isdir(os_path): listing = os.listdir(os_path) # Don't permanently delete non-empty directories. # A directory containing only leftover checkpoints is # considered empty. cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None) for entry in listing: if entry != cp_dir: raise web.HTTPError(400, u'Directory %s not empty' % os_path) self.log.debug("Removing directory %s", os_path) with self.perm_to_403(): shutil.rmtree(os_path) else: self.log.debug("Unlinking file %s", os_path) with self.perm_to_403(): rm(os_path) def rename_file(self, old_path, new_path): """Rename a file.""" old_path = old_path.strip('/') new_path = new_path.strip('/') if new_path == old_path: return new_os_path = self._get_os_path(new_path) old_os_path = self._get_os_path(old_path) # Should we proceed with the move? if os.path.exists(new_os_path) and not samefile( old_os_path, new_os_path): raise web.HTTPError(409, u'File already exists: %s' % new_path) # Move the file try: with self.perm_to_403(): shutil.move(old_os_path, new_os_path) except web.HTTPError: raise except Exception as e: raise web.HTTPError( 500, u'Unknown error renaming file: %s %s' % (old_path, e)) def info_string(self): return _("Serving notebooks from local directory: %s") % self.root_dir def get_kernel_path(self, path, model=None): """Return the initial API path of a kernel associated with a given notebook""" if self.dir_exists(path): return path if '/' in path: parent_dir = path.rsplit('/', 1)[0] else: parent_dir = '' return parent_dir
class ZMQDisplayPublisher(DisplayPublisher): """A display publisher that publishes data using a ZeroMQ PUB socket.""" session = Instance(Session, allow_none=True) pub_socket = Any(allow_none=True) parent_header = Dict({}) topic = CBytes(b'display_data') # thread_local: # An attribute used to ensure the correct output message # is processed. See ibpykernel Issue 113 for a discussion. _thread_local = Any() def set_parent(self, parent): """Set the parent for outbound messages.""" self.parent_header = extract_header(parent) def _flush_streams(self): """flush IO Streams prior to display""" sys.stdout.flush() sys.stderr.flush() @default('_thread_local') def _default_thread_local(self): """Initialize our thread local storage""" return local() @property def _hooks(self): if not hasattr(self._thread_local, 'hooks'): # create new list for a new thread self._thread_local.hooks = [] return self._thread_local.hooks def publish( self, data, metadata=None, source=None, transient=None, update=False, ): """Publish a display-data message Parameters ---------- data: dict A mime-bundle dict, keyed by mime-type. metadata: dict, optional Metadata associated with the data. transient: dict, optional, keyword-only Transient data that may only be relevant during a live display, such as display_id. Transient data should not be persisted to documents. update: bool, optional, keyword-only If True, send an update_display_data message instead of display_data. """ self._flush_streams() if metadata is None: metadata = {} if transient is None: transient = {} self._validate_data(data, metadata) content = {} content['data'] = encode_images(data) content['metadata'] = metadata content['transient'] = transient msg_type = 'update_display_data' if update else 'display_data' # Use 2-stage process to send a message, # in order to put it through the transform # hooks before potentially sending. msg = self.session.msg(msg_type, json_clean(content), parent=self.parent_header) # Each transform either returns a new # message or None. If None is returned, # the message has been 'used' and we return. for hook in self._hooks: msg = hook(msg) if msg is None: return self.session.send( self.pub_socket, msg, ident=self.topic, ) def clear_output(self, wait=False): """Clear output associated with the current execution (cell). Parameters ---------- wait: bool (default: False) If True, the output will not be cleared immediately, instead waiting for the next display before clearing. This reduces bounce during repeated clear & display loops. """ content = dict(wait=wait) self._flush_streams() self.session.send( self.pub_socket, u'clear_output', content, parent=self.parent_header, ident=self.topic, ) def register_hook(self, hook): """ Registers a hook with the thread-local storage. Parameters ---------- hook : Any callable object Returns ------- Either a publishable message, or `None`. The DisplayHook objects must return a message from the __call__ method if they still require the `session.send` method to be called after tranformation. Returning `None` will halt that execution path, and session.send will not be called. """ self._hooks.append(hook) def unregister_hook(self, hook): """ Un-registers a hook with the thread-local storage. Parameters ---------- hook: Any callable object which has previously been registered as a hook. Returns ------- bool - `True` if the hook was removed, `False` if it wasn't found. """ try: self._hooks.remove(hook) return True except ValueError: return False
class GFKernel(Kernel): implementation = 'GF' implementation_version = '1.0' language = 'gf' language_version = '0.1' # TODO change this to gf language_info = { 'name': 'gf', 'mimetype': 'text/gf', 'file_extension': '.gf', } banner = "GF" shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) shell_class = Type(ZMQInteractiveShell) use_experimental_completions = Bool( True, help= "Set this flag to False to deactivate the use of experimental IPython completion APIs.", ).tag(config=True) user_module = Any() def _user_module_changed(self, name, old, new): if self.shell is not None: self.shell.user_module = new user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() def __init__(self, **kwargs): super(GFKernel, self).__init__(**kwargs) # set up the shell self.shell = self.shell_class.instance( parent=self, profile_dir=self.profile_dir, user_module=self.user_module, user_ns=self.user_ns, kernel=self, ) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket self.shell.displayhook.topic = self._topic('execute_result') self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket # set up and attach comm_manager to the shell self.comm_manager = CommManager(parent=self, kernel=self) self.shell.configurables.append(self.comm_manager) comm_msg_types = ['comm_open', 'comm_msg', 'comm_close'] for msg_type in comm_msg_types: self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type) # initialize the GFRepl self.GFRepl = GFRepl() def start(self): self.shell.exit_now = False super(GFKernel, self).start() def set_parent(self, ident, parent): """Overridden from parent to tell the display hook and output streams about the parent message. """ super(GFKernel, self).set_parent(ident, parent) self.shell.set_parent(parent) def init_metadata(self, parent): """Initialize metadata. Run at the beginning of each execution request. """ md = super(GFKernel, self).init_metadata(parent) # FIXME: remove deprecated ipyparallel-specific code # This is required for ipyparallel < 5.0 md.update({ 'dependencies_met': True, 'engine': self.ident, }) return md def do_execute(self, code, silent=False, store_history=True, user_expressions=None, allow_stdin=True): """Called when the user inputs code""" # img_data = Image.open('/home/kai/gf_content/out.png','r') messages = self.GFRepl.handle_input(code) for msg in messages: if msg['file']: file_name = msg['file'] try: with open(file_name, "rb") as f: img = f.read() display(widgets.Image(value=img, format='png')) except: self.send_response( self.iopub_socket, 'display_data', to_display_data("There is no tree to show!")) elif msg['message']: self.send_response(self.iopub_socket, 'display_data', to_display_data(msg['message'])) elif msg['trees']: dd = widgets.Dropdown( layout={'width': 'max-content'}, options=msg['trees'], value=msg['trees'][0], description='Tree of:', disabled=False, ) file_name = self.GFRepl.handle_single_view( "%s %s" % (msg['tree_type'], msg['trees'][0])) with open(file_name, "rb") as f: img = f.read() image = widgets.Image(value=img, format='png') def on_value_change(change): file_name = self.GFRepl.handle_single_view( "%s %s" % (msg['tree_type'], change['new'])) with open(file_name, "rb") as f: img = f.read() image.value = img dd.observe(on_value_change, names='value') display(dd, image) return { 'status': 'ok', # The base class increments the execution count 'payload': [], 'execution_count': self.execution_count, 'user_expressions': {}, } def do_shutdown(self, restart): """Called when the kernel is terminated""" self.GFRepl.do_shutdown() def do_complete(self, code, cursorPos): """Autocompletion when the user presses tab""" # load the shortcuts from the unicode-latex-map last_word = get_current_word(code, cursorPos) matches = get_matches(last_word) if not last_word or not matches: matches = None return { 'matches': matches, 'cursor_start': cursorPos - len(last_word), 'cursor_end': cursorPos, 'metadata': {}, 'status': 'ok' }
class S3FS(GenericFS): access_key_id = Unicode(help="S3/AWS access key ID", allow_none=True, default_value=None).tag( config=True, env="JPYNB_S3_ACCESS_KEY_ID") secret_access_key = Unicode(help="S3/AWS secret access key", allow_none=True, default_value=None).tag( config=True, env="JPYNB_S3_SECRET_ACCESS_KEY") endpoint_url = Unicode("s3.amazonaws.com", help="S3 endpoint URL").tag( config=True, env="JPYNB_S3_ENDPOINT_URL") region_name = Unicode("us-east-1", help="Region name").tag(config=True, env="JPYNB_S3_REGION_NAME") bucket = Unicode("notebooks", help="Bucket name to store notebooks").tag( config=True, env="JPYNB_S3_BUCKET") signature_version = Unicode(help="").tag(config=True) sse = Unicode(help="Type of server-side encryption to use").tag( config=True) kms_key_id = Unicode(help="KMS ID to use to encrypt workbooks").tag( config=True) prefix = Unicode( "", help="Prefix path inside the specified bucket").tag(config=True) delimiter = Unicode("/", help="Path delimiter").tag(config=True) dir_keep_file = Unicode( ".s3keep", help="Empty file to create when creating directories").tag(config=True) session_token = Unicode(help="S3/AWS session token", allow_none=True, default_value=None).tag( config=True, env="JPYNB_S3_SESSION_TOKEN") boto3_session = Any( help="Place to store customer boto3 session instance - likely passed in" ) def __init__(self, log, **kwargs): super(S3FS, self).__init__(**kwargs) self.log = log client_kwargs = { "endpoint_url": self.endpoint_url, "region_name": self.region_name, } config_kwargs = {} if self.signature_version: config_kwargs["signature_version"] = self.signature_version s3_additional_kwargs = {} if self.sse: s3_additional_kwargs["ServerSideEncryption"] = self.sse if self.kms_key_id: s3_additional_kwargs["SSEKMSKeyId"] = self.kms_key_id self.fs = s3fs.S3FileSystem( key=self.access_key_id, secret=self.secret_access_key, token=self.session_token, client_kwargs=client_kwargs, config_kwargs=config_kwargs, s3_additional_kwargs=s3_additional_kwargs, session=self.boto3_session, ) self.init() def init(self): try: self.mkdir("") self.ls("") self.isdir("") except ClientError as ex: if "AccessDenied" in str(ex): policy = SAMPLE_ACCESS_POLICY.format( bucket=os.path.join(self.bucket, self.prefix)) self.log.error( "AccessDenied error while creating initial S3 objects. Create an IAM policy like:\n{policy}" .format(policy=policy)) sys.exit(1) else: raise ex # GenericFS methods ----------------------------------------------------------------------------------------------- def ls(self, path=""): path_ = self.path(path) self.log.debug("S3contents.S3FS: Listing directory: `%s`", path_) files = self.fs.ls(path_, refresh=True) return self.unprefix(files) def isfile(self, path): path_ = self.path(path) # FileNotFoundError handled by s3fs is_file = self.fs.isfile(path_) self.log.debug("S3contents.S3FS: `%s` is a file: %s", path_, is_file) return is_file def isdir(self, path): path_ = self.path(path) # FileNotFoundError handled by s3fs is_dir = self.fs.isdir(path_) self.log.debug("S3contents.S3FS: `%s` is a directory: %s", path_, is_dir) return is_dir def mv(self, old_path, new_path): self.log.debug("S3contents.S3FS: Move file `%s` to `%s`", old_path, new_path) self.cp(old_path, new_path) self.rm(old_path) def cp(self, old_path, new_path): old_path_, new_path_ = self.path(old_path), self.path(new_path) self.log.debug("S3contents.S3FS: Coping `%s` to `%s`", old_path_, new_path_) if self.isdir(old_path): old_dir_path, new_dir_path = old_path, new_path for obj in self.ls(old_dir_path): old_item_path = obj new_item_path = old_item_path.replace(old_dir_path, new_dir_path, 1) self.cp(old_item_path, new_item_path) self.mkdir(new_path) # Touch with dir_keep_file elif self.isfile(old_path): self.fs.copy(old_path_, new_path_) def rm(self, path): path_ = self.path(path) self.log.debug("S3contents.S3FS: Removing: `%s`", path_) if self.isfile(path): self.log.debug("S3contents.S3FS: Removing file: `%s`", path_) self.fs.rm(path_) elif self.isdir(path): self.log.debug("S3contents.S3FS: Removing directory: `%s`", path_) self.fs.rm(path_ + self.delimiter, recursive=True) # self.fs.rmdir(path_ + self.delimiter, recursive=True) def mkdir(self, path): path_ = self.path(path, self.dir_keep_file) self.log.debug("S3contents.S3FS: Making dir: `%s`", path_) self.fs.touch(path_) def read(self, path, format): path_ = self.path(path) if not self.isfile(path): raise NoSuchFile(path_) with self.fs.open(path_, mode="rb") as f: content = f.read() if format is None or format == "text": # Try to interpret as unicode if format is unknown or if unicode # was explicitly requested. try: return content.decode("utf-8"), "text" except UnicodeError: if format == "text": err = "{} is not UTF-8 encoded".format(path_) self.log.error(err) raise HTTPError(400, err, reason="bad format") return base64.b64encode(content).decode("ascii"), "base64" def lstat(self, path): path_ = self.path(path) if self.fs.isdir(path_): # Try to get status of the dir_keep_file path_ = self.path(path, self.dir_keep_file) try: self.fs.invalidate_cache(path_) info = self.fs.info(path_) except FileNotFoundError: return {"ST_MTIME": None} ret = {} ret["ST_MTIME"] = info["LastModified"] return ret def write(self, path, content, format): path_ = self.path(self.unprefix(path)) self.log.debug("S3contents.S3FS: Writing file: `%s`", path_) if format not in {"text", "base64"}: raise HTTPError( 400, "Must specify format of file contents as 'text' or 'base64'", ) try: if format == "text": content_ = content.encode("utf8") else: b64_bytes = content.encode("ascii") content_ = base64.b64decode(b64_bytes) except Exception as e: raise HTTPError(400, "Encoding error saving %s: %s" % (path_, e)) with self.fs.open(path_, mode="wb") as f: f.write(content_) def writenotebook(self, path, content): path_ = self.path(self.unprefix(path)) self.log.debug("S3contents.S3FS: Writing notebook: `%s`", path_) with self.fs.open(path_, mode="wb") as f: f.write(content.encode("utf-8")) # Utilities ------------------------------------------------------------------------------------------------------- def get_prefix(self): """Full prefix: bucket + optional prefix""" prefix = self.bucket if self.prefix: prefix += self.delimiter + self.prefix return prefix prefix_ = property(get_prefix) def unprefix(self, path): """Remove the self.prefix_ (if present) from a path or list of paths""" if isinstance(path, str): path = path[len(self.prefix_):] if path.startswith( self.prefix_) else path path = path[1:] if path.startswith(self.delimiter) else path return path if isinstance(path, (list, tuple)): path = [ p[len(self.prefix_):] if p.startswith(self.prefix_) else p for p in path ] path = [p[1:] if p.startswith(self.delimiter) else p for p in path] return path def path(self, *path): """Utility to join paths including the bucket and prefix""" path = list(filter(None, path)) path = self.unprefix(path) items = [self.prefix_] + path return self.delimiter.join(items)
class ImageCollectionBrowser(HasTraits): paths = Union((List(), Unicode())) file_index = Int(0) path = Unicode() images = Array() image = Any() frame_index = Int(0) num_frames = Int(0) binning = Int(1) def __init__(self, **kwargs): self._path_text = widgets.HTML(description='File name') self._file_index_slider = IntSliderWithButtons( description='File index', min=0) self._frame_index_slider = IntSliderWithButtons( description='Frame index', min=0) link((self._file_index_slider, 'value'), (self, 'file_index')) link((self._frame_index_slider, 'value'), (self, 'frame_index')) super().__init__(**kwargs) @observe('frame_index') def _observe_frame_index(self, *args): image = self.images[self.frame_index] if self.binning != 1: image = downscale_local_mean(image, factors=(self.binning, ) * 2) self.image = image @observe('path') def _observe_path(self, *args): self._path_text.value = str(Path(*Path(self.path).parts[-4:])) images = imread(self.path) # if images.shape[-1] in (3, 4): # images = np.swapaxes(images, 0, 2) if len(images.shape) == 2: images = images[None] assert len(images.shape) == 3 self.images = images self.num_frames = len(self.images) self._frame_index_slider.max = len(self.images) - 1 if self.frame_index == 0: self._observe_frame_index() else: self.frame_index = 0 @observe('file_index') def _observe_file_index(self, *args): self.path = str(self.paths[self.file_index]) @observe('paths') def _observe_filename(self, change): self._file_index_slider.max = len(self.paths) - 1 if self.file_index == 0: self._observe_file_index() else: self.file_index = 0 @property def widgets(self): hbox = widgets.VBox([ self._path_text, self._file_index_slider, self._frame_index_slider ]) return hbox
class ContentsManager(LoggingConfigurable): """Base class for serving files and directories. This serves any text or binary file, as well as directories, with special handling for JSON notebook documents. Most APIs take a path argument, which is always an API-style unicode path, and always refers to a directory. - unicode, not url-escaped - '/'-separated - leading and trailing '/' will be stripped - if unspecified, path defaults to '', indicating the root path. """ root_dir = Unicode('/', config=True) allow_hidden = Bool(False, config=True, help="Allow access to hidden files") notary = Instance(sign.NotebookNotary) def _notary_default(self): return sign.NotebookNotary(parent=self) hide_globs = List(Unicode(), [ u'__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~', ], config=True, help=""" Glob patterns to hide in file and directory listings. """) untitled_notebook = Unicode( _("Untitled"), config=True, help="The base name used when creating untitled notebooks.") untitled_file = Unicode( "untitled", config=True, help="The base name used when creating untitled files.") untitled_directory = Unicode( "Untitled Folder", config=True, help="The base name used when creating untitled directories.") pre_save_hook = Any(None, config=True, allow_none=True, help="""Python callable or importstring thereof To be called on a contents model prior to save. This can be used to process the structure, such as removing notebook outputs or other side effects that should not be saved. It will be called as (all arguments passed by keyword):: hook(path=path, model=model, contents_manager=self) - model: the model to be saved. Includes file contents. Modifying this dict will affect the file that is stored. - path: the API path of the save destination - contents_manager: this ContentsManager instance """) @validate('pre_save_hook') def _validate_pre_save_hook(self, proposal): value = proposal['value'] if isinstance(value, string_types): value = import_item(self.pre_save_hook) if not callable(value): raise TraitError("pre_save_hook must be callable") return value def run_pre_save_hook(self, model, path, **kwargs): """Run the pre-save hook if defined, and log errors""" if self.pre_save_hook: try: self.log.debug("Running pre-save hook on %s", path) self.pre_save_hook(model=model, path=path, contents_manager=self, **kwargs) except Exception: self.log.error("Pre-save hook failed on %s", path, exc_info=True) checkpoints_class = Type(Checkpoints, config=True) checkpoints = Instance(Checkpoints, config=True) checkpoints_kwargs = Dict(config=True) @default('checkpoints') def _default_checkpoints(self): return self.checkpoints_class(**self.checkpoints_kwargs) @default('checkpoints_kwargs') def _default_checkpoints_kwargs(self): return dict( parent=self, log=self.log, ) files_handler_class = Type( FilesHandler, klass=RequestHandler, allow_none=True, config=True, help="""handler class to use when serving raw file requests. Default is a fallback that talks to the ContentsManager API, which may be inefficient, especially for large files. Local files-based ContentsManagers can use a StaticFileHandler subclass, which will be much more efficient. Access to these files should be Authenticated. """) files_handler_params = Dict( config=True, help="""Extra parameters to pass to files_handler_class. For example, StaticFileHandlers generally expect a `path` argument specifying the root directory from which to serve files. """) def get_extra_handlers(self): """Return additional handlers Default: self.files_handler_class on /files/.* """ handlers = [] if self.files_handler_class: handlers.append((r"/files/(.*)", self.files_handler_class, self.files_handler_params)) return handlers # ContentsManager API part 1: methods that must be # implemented in subclasses. def dir_exists(self, path): """Does a directory exist at the given path? Like os.path.isdir Override this method in subclasses. Parameters ---------- path : string The path to check Returns ------- exists : bool Whether the path does indeed exist. """ raise NotImplementedError def is_hidden(self, path): """Is path a hidden directory or file? Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to root dir). Returns ------- hidden : bool Whether the path is hidden. """ raise NotImplementedError def file_exists(self, path=''): """Does a file exist at the given path? Like os.path.isfile Override this method in subclasses. Parameters ---------- path : string The API path of a file to check for. Returns ------- exists : bool Whether the file exists. """ raise NotImplementedError('must be implemented in a subclass') def exists(self, path): """Does a file or directory exist at the given path? Like os.path.exists Parameters ---------- path : string The API path of a file or directory to check for. Returns ------- exists : bool Whether the target exists. """ return self.file_exists(path) or self.dir_exists(path) def get(self, path, content=True, type=None, format=None): """Get a file or directory model.""" raise NotImplementedError('must be implemented in a subclass') def save(self, model, path): """ Save a file or directory model to path. Should return the saved model with no content. Save implementations should call self.run_pre_save_hook(model=model, path=path) prior to writing any data. """ raise NotImplementedError('must be implemented in a subclass') def delete_file(self, path): """Delete the file or directory at path.""" raise NotImplementedError('must be implemented in a subclass') def rename_file(self, old_path, new_path): """Rename a file or directory.""" raise NotImplementedError('must be implemented in a subclass') # ContentsManager API part 2: methods that have useable default # implementations, but can be overridden in subclasses. def delete(self, path): """Delete a file/directory and any associated checkpoints.""" path = path.strip('/') if not path: raise HTTPError(400, "Can't delete root") self.delete_file(path) self.checkpoints.delete_all_checkpoints(path) def rename(self, old_path, new_path): """Rename a file and any checkpoints associated with that file.""" self.rename_file(old_path, new_path) self.checkpoints.rename_all_checkpoints(old_path, new_path) def update(self, model, path): """Update the file's path For use in PATCH requests, to enable renaming a file without re-uploading its contents. Only used for renaming at the moment. """ path = path.strip('/') new_path = model.get('path', path).strip('/') if path != new_path: self.rename(path, new_path) model = self.get(new_path, content=False) return model def info_string(self): return "Serving contents" def get_kernel_path(self, path, model=None): """Return the API path for the kernel KernelManagers can turn this value into a filesystem path, or ignore it altogether. The default value here will start kernels in the directory of the notebook server. FileContentsManager overrides this to use the directory containing the notebook. """ return '' def increment_filename(self, filename, path='', insert=''): """Increment a filename until it is unique. Parameters ---------- filename : unicode The name of a file, including extension path : unicode The API path of the target's directory insert: unicode The characters to insert after the base filename Returns ------- name : unicode A filename that is unique, based on the input filename. """ # Extract the full suffix from the filename (e.g. .tar.gz) path = path.strip('/') basename, dot, ext = filename.partition('.') suffix = dot + ext for i in itertools.count(): if i: insert_i = '{}{}'.format(insert, i) else: insert_i = '' name = u'{basename}{insert}{suffix}'.format(basename=basename, insert=insert_i, suffix=suffix) if not self.exists(u'{}/{}'.format(path, name)): break return name def validate_notebook_model(self, model): """Add failed-validation message to model""" try: validate_nb(model['content']) except ValidationError as e: model['message'] = u'Notebook validation failed: {}:\n{}'.format( e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'), ) return model def new_untitled(self, path='', type='', ext=''): """Create a new untitled file or directory in path path must be a directory File extension can be specified. Use `new` to create files with a fully specified path (including filename). """ path = path.strip('/') if not self.dir_exists(path): raise HTTPError(404, 'No such directory: %s' % path) model = {} if type: model['type'] = type if ext == '.ipynb': model.setdefault('type', 'notebook') else: model.setdefault('type', 'file') insert = '' if model['type'] == 'directory': untitled = self.untitled_directory insert = ' ' elif model['type'] == 'notebook': untitled = self.untitled_notebook ext = '.ipynb' elif model['type'] == 'file': untitled = self.untitled_file else: raise HTTPError(400, "Unexpected model type: %r" % model['type']) name = self.increment_filename(untitled + ext, path, insert=insert) path = u'{0}/{1}'.format(path, name) return self.new(model, path) def new(self, model=None, path=''): """Create a new file or directory and return its model with no content. To create a new untitled entity in a directory, use `new_untitled`. """ path = path.strip('/') if model is None: model = {} if path.endswith('.ipynb'): model.setdefault('type', 'notebook') else: model.setdefault('type', 'file') # no content, not a directory, so fill out new-file model if 'content' not in model and model['type'] != 'directory': if model['type'] == 'notebook': model['content'] = new_notebook() model['format'] = 'json' else: model['content'] = '' model['type'] = 'file' model['format'] = 'text' model = self.save(model, path) return model def copy(self, from_path, to_path=None): """Copy an existing file and return its new model. If to_path not specified, it will be the parent directory of from_path. If to_path is a directory, filename will increment `from_path-Copy#.ext`. from_path must be a full path to a file. """ path = from_path.strip('/') if to_path is not None: to_path = to_path.strip('/') if '/' in path: from_dir, from_name = path.rsplit('/', 1) else: from_dir = '' from_name = path model = self.get(path) model.pop('path', None) model.pop('name', None) if model['type'] == 'directory': raise HTTPError(400, "Can't copy directories") if to_path is None: to_path = from_dir if self.dir_exists(to_path): name = copy_pat.sub(u'.', from_name) to_name = self.increment_filename(name, to_path, insert='-Copy') to_path = u'{0}/{1}'.format(to_path, to_name) model = self.save(model, to_path) return model def log_info(self): self.log.info(self.info_string()) def trust_notebook(self, path): """Explicitly trust a notebook Parameters ---------- path : string The path of a notebook """ model = self.get(path) nb = model['content'] self.log.warning("Trusting notebook %s", path) self.notary.mark_cells(nb, True) self.check_and_sign(nb, path) def check_and_sign(self, nb, path=''): """Check for trusted cells, and sign the notebook. Called as a part of saving notebooks. Parameters ---------- nb : dict The notebook dict path : string The notebook's path (for logging) """ if self.notary.check_cells(nb): self.notary.sign(nb) else: self.log.warning("Notebook %s is not trusted", path) def mark_trusted_cells(self, nb, path=''): """Mark cells as trusted if the notebook signature matches. Called as a part of loading notebooks. Parameters ---------- nb : dict The notebook object (in current nbformat) path : string The notebook's path (for logging) """ trusted = self.notary.check_signature(nb) if not trusted: self.log.warning("Notebook %s is not trusted", path) self.notary.mark_cells(nb, trusted) def should_list(self, name): """Should this file/directory name be displayed in a listing?""" return not any(fnmatch(name, glob) for glob in self.hide_globs) # Part 3: Checkpoints API def create_checkpoint(self, path): """Create a checkpoint.""" return self.checkpoints.create_checkpoint(self, path) def restore_checkpoint(self, checkpoint_id, path): """ Restore a checkpoint. """ self.checkpoints.restore_checkpoint(self, checkpoint_id, path) def list_checkpoints(self, path): return self.checkpoints.list_checkpoints(path) def delete_checkpoint(self, checkpoint_id, path): return self.checkpoints.delete_checkpoint(checkpoint_id, path)
class OAuthenticator(Authenticator): """Base class for OAuthenticators Subclasses must override: login_service (string identifying the service provider) authenticate (method takes one arg - the request handler handling the oauth callback) """ login_handler = OAuthLoginHandler callback_handler = OAuthCallbackHandler logout_handler = OAuthLogoutHandler authorize_url = Unicode( config=True, help="""The authenticate url for initiating oauth""" ) @default("authorize_url") def _authorize_url_default(self): return os.environ.get("OAUTH2_AUTHORIZE_URL", "") token_url = Unicode( config=True, help="""The url retrieving an access token at the completion of oauth""", ) @default("token_url") def _token_url_default(self): return os.environ.get("OAUTH2_TOKEN_URL", "") userdata_url = Unicode( config=True, help="""The url for retrieving user data with a completed access token""", ) @default("userdata_url") def _userdata_url_default(self): return os.environ.get("OAUTH2_USERDATA_URL", "") scope = List( Unicode(), config=True, help="""The OAuth scopes to request. See the OAuth documentation of your OAuth provider for options. For GitHub in particular, you can see github_scopes.md in this repo. """, ) extra_authorize_params = Dict( config=True, help="""Extra GET params to send along with the initial OAuth request to the OAuth provider.""", ) login_service = 'override in subclass' oauth_callback_url = Unicode( os.getenv('OAUTH_CALLBACK_URL', ''), config=True, help="""Callback URL to use. Typically `https://{host}/hub/oauth_callback`""", ) client_id_env = '' client_id = Unicode(config=True) def _client_id_default(self): if self.client_id_env: client_id = os.getenv(self.client_id_env, '') if client_id: return client_id return os.getenv('OAUTH_CLIENT_ID', '') client_secret_env = '' client_secret = Unicode(config=True) def _client_secret_default(self): if self.client_secret_env: client_secret = os.getenv(self.client_secret_env, '') if client_secret: return client_secret return os.getenv('OAUTH_CLIENT_SECRET', '') validate_server_cert_env = 'OAUTH_TLS_VERIFY' validate_server_cert = Bool(config=True) def _validate_server_cert_default(self): env_value = os.getenv(self.validate_server_cert_env, '') if env_value == '0': return False else: return True http_client = Any() @default("http_client") def _default_http_client(self): return AsyncHTTPClient() async def fetch(self, req, label="fetching", parse_json=True, **kwargs): """Wrapper for http requests logs error responses, parses successful JSON responses Args: req: tornado HTTPRequest label (str): label describing what is happening, used in log message when the request fails. **kwargs: remaining keyword args passed to underlying `client.fetch(req, **kwargs)` Returns: r: parsed JSON response """ try: resp = await self.http_client.fetch(req, **kwargs) except HTTPClientError as e: if e.response: # Log failed response message for debugging purposes message = e.response.body.decode("utf8", "replace") try: # guess json, reformat for readability json_message = json.loads(message) except ValueError: # not json pass else: # reformat json log message for readability message = json.dumps(json_message, sort_keys=True, indent=1) else: # didn't get a response, e.g. connection error message = str(e) # log url without query params url = urlunparse(urlparse(req.url)._replace(query="")) app_log.error(f"Error {label} {e.code} {req.method} {url}: {message}") raise e else: if parse_json: if resp.body: return json.loads(resp.body.decode('utf8', 'replace')) else: # empty body is None return None else: return resp def login_url(self, base_url): return url_path_join(base_url, 'oauth_login') def logout_url(self, base_url): return url_path_join(base_url, 'logout') def get_callback_url(self, handler=None): """Get my OAuth redirect URL Either from config or guess based on the current request. """ if self.oauth_callback_url: return self.oauth_callback_url elif handler: return guess_callback_uri( handler.request.protocol, handler.request.host, handler.hub.server.base_url, ) else: raise ValueError( "Specify callback oauth_callback_url or give me a handler to guess with" ) def get_handlers(self, app): return [ (r'/oauth_login', self.login_handler), (r'/oauth_callback', self.callback_handler), (r'/logout', self.logout_handler), ] async def authenticate(self, handler, data=None): raise NotImplementedError() _deprecated_oauth_aliases = {} def _deprecated_oauth_trait(self, change): """observer for deprecated traits""" old_attr = change.name new_attr, version = self._deprecated_oauth_aliases.get(old_attr) new_value = getattr(self, new_attr) if new_value != change.new: # only warn if different # protects backward-compatible config from warnings # if they set the same value under both names self.log.warning( "{cls}.{old} is deprecated in {cls} {version}, use {cls}.{new} instead".format( cls=self.__class__.__name__, old=old_attr, new=new_attr, version=version, ) ) setattr(self, new_attr, change.new) def __init__(self, **kwargs): # observe deprecated config names in oauthenticator if self._deprecated_oauth_aliases: self.observe( self._deprecated_oauth_trait, names=list(self._deprecated_oauth_aliases) ) super().__init__(**kwargs)
class TraefikProxy(Proxy): """JupyterHub Proxy implementation using traefik""" traefik_process = Any() toml_static_config_file = Unicode( "traefik.toml", config=True, help="""traefik's static configuration file""" ) traefik_api_url = Unicode( "http://127.0.0.1:8099", config=True, help="""traefik authenticated api endpoint url""", ) traefik_log_level = Unicode("ERROR", config=True, help="""traefik's log level""") traefik_api_password = Unicode( config=True, help="""The password for traefik api login""" ) @default("traefik_api_password") def _warn_empty_password(self): self.log.warning("Traefik API password was not set.") if self.should_start: # Generating tokens is fine if the Hub is starting the proxy self.log.warning("Generating a random token for traefik_api_username...") return new_token() self.log.warning( "Please set c.TraefikProxy.traefik_api_password to authenticate with traefik" " if the proxy was not started by the Hub." ) return "" traefik_api_username = Unicode( config=True, help="""The username for traefik api login""" ) @default("traefik_api_username") def _warn_empty_username(self): self.log.warning("Traefik API username was not set.") if self.should_start: self.log.warning('Defaulting traefik_api_username to "jupyterhub"') return "jupyterhub" self.log.warning( "Please set c.TraefikProxy.traefik_api_username to authenticate with traefik" " if the proxy was not started by the Hub." ) return "" traefik_api_hashed_password = Unicode() check_route_timeout = Integer( 30, config=True, help="""Timeout (in seconds) when waiting for traefik to register an updated route.""", ) static_config = Dict() def _generate_htpassword(self): from passlib.apache import HtpasswdFile ht = HtpasswdFile() ht.set_password(self.traefik_api_username, self.traefik_api_password) self.traefik_api_hashed_password = str(ht.to_string()).split(":")[1][:-3] async def _check_for_traefik_endpoint(self, routespec, kind, provider): """Check for an expected frontend or backend This is used to wait for traefik to load configuration from a provider """ expected = traefik_utils.generate_alias(routespec, kind) path = "/api/providers/{}/{}s".format(provider, kind) try: resp = await self._traefik_api_request(path) data = json.loads(resp.body) except Exception: self.log.exception("Error checking traefik api for %s %s", kind, routespec) return False if expected not in data: self.log.debug("traefik %s not yet in %ss", expected, kind) self.log.debug("Current traefik %ss: %s", kind, data) return False # found the expected endpoint return True async def _wait_for_route(self, routespec, provider): self.log.info("Waiting for %s to register with traefik", routespec) async def _check_traefik_dynamic_conf_ready(): """Check if traefik loaded its dynamic configuration yet""" if not await self._check_for_traefik_endpoint( routespec, "backend", provider ): return False if not await self._check_for_traefik_endpoint( routespec, "frontend", provider ): return False return True await exponential_backoff( _check_traefik_dynamic_conf_ready, "Traefik route for %s configuration not available" % routespec, timeout=self.check_route_timeout, ) async def _traefik_api_request(self, path): """Make an API request to traefik""" url = url_path_join(self.traefik_api_url, path) self.log.debug("Fetching traefik api %s", url) resp = await AsyncHTTPClient().fetch( url, auth_username=self.traefik_api_username, auth_password=self.traefik_api_password, ) if resp.code >= 300: self.log.warning("%s GET %s", resp.code, url) else: self.log.debug("%s GET %s", resp.code, url) return resp async def _wait_for_static_config(self, provider): async def _check_traefik_static_conf_ready(): """ Check if traefik loaded its static configuration from the etcd cluster """ try: resp = await self._traefik_api_request("/api/providers/" + provider) except Exception: self.log.exception("Error checking for traefik static configuration") return False if resp.code != 200: self.log.error( "Unexpected response code %s checking for traefik static configuration", resp.code, ) return False return True await exponential_backoff( _check_traefik_static_conf_ready, "Traefik static configuration not available", timeout=self.check_route_timeout, ) def _stop_traefik(self): self.log.info("Cleaning up proxy[%i]...", self.traefik_process.pid) self.traefik_process.kill() self.traefik_process.wait() def _launch_traefik(self, config_type): if config_type == "toml" or config_type == "etcdv3" or config_type == "consul": config_file_path = abspath(join(dirname(__file__), "traefik.toml")) self.traefik_process = Popen( ["traefik", "-c", config_file_path], stdout=None ) else: raise ValueError( "Configuration mode not supported \n.\ The proxy can only be configured through toml and etcd" ) async def _setup_traefik_static_config(self): self.log.info("Setting up traefik's static config...") self._generate_htpassword() self.static_config = {} self.static_config["defaultentrypoints"] = ["http"] self.static_config["debug"] = True self.static_config["logLevel"] = self.traefik_log_level entryPoints = {} entryPoints["http"] = {"address": ":" + str(urlparse(self.public_url).port)} auth = { "basic": { "users": [ self.traefik_api_username + ":" + self.traefik_api_hashed_password ] } } entryPoints["auth_api"] = { "address": ":" + str(urlparse(self.traefik_api_url).port), "auth": auth, } self.static_config["entryPoints"] = entryPoints self.static_config["api"] = {"dashboard": True, "entrypoint": "auth_api"} self.static_config["wss"] = {"protocol": "http"} async def start(self): """Start the proxy. Will be called during startup if should_start is True. **Subclasses must define this method** if the proxy is to be started by the Hub """ self._start_traefik() await self._setup_traefik_static_config() async def stop(self): """Stop the proxy. Will be called during teardown if should_start is True. **Subclasses must define this method** if the proxy is to be started by the Hub """ self._stop_traefik() async def add_route(self, routespec, target, data): """Add a route to the proxy. **Subclasses must define this method** Args: routespec (str): A URL prefix ([host]/path/) for which this route will be matched, e.g. host.name/path/ target (str): A full URL that will be the target of this route. data (dict): A JSONable dict that will be associated with this route, and will be returned when retrieving information about this route. Will raise an appropriate Exception (FIXME: find what?) if the route could not be added. The proxy implementation should also have a way to associate the fact that a route came from JupyterHub. """ raise NotImplementedError() async def delete_route(self, routespec): """Delete a route with a given routespec if it exists. **Subclasses must define this method** """ raise NotImplementedError() async def get_all_routes(self): """Fetch and return all the routes associated by JupyterHub from the proxy. **Subclasses must define this method** Should return a dictionary of routes, where the keys are routespecs and each value is a dict of the form:: { 'routespec': the route specification ([host]/path/) 'target': the target host URL (proto://host) for this route 'data': the attached data dict for this route (as specified in add_route) } """ raise NotImplementedError() async def get_route(self, routespec): """Return the route info for a given routespec. Args: routespec (str): A URI that was used to add this route, e.g. `host.tld/path/` Returns: result (dict): dict with the following keys:: 'routespec': The normalized route specification passed in to add_route ([host]/path/) 'target': The target host for this route (proto://host) 'data': The arbitrary data dict that was passed in by JupyterHub when adding this route. None: if there are no routes matching the given routespec """ raise NotImplementedError()
class IPythonKernel(KernelBase): shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) shell_class = Type(ZMQInteractiveShell) use_experimental_completions = Bool(True, help="Set this flag to False to deactivate the use of experimental IPython completion APIs.", ).tag(config=True) debugpy_stream = Instance(ZMQStream, allow_none=True) user_module = Any() @observe('user_module') @observe_compat def _user_module_changed(self, change): if self.shell is not None: self.shell.user_module = change['new'] user_ns = Instance(dict, args=None, allow_none=True) @observe('user_ns') @observe_compat def _user_ns_changed(self, change): if self.shell is not None: self.shell.user_ns = change['new'] self.shell.init_user_ns() # A reference to the Python builtin 'raw_input' function. # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3) _sys_raw_input = Any() _sys_eval_input = Any() def __init__(self, **kwargs): super(IPythonKernel, self).__init__(**kwargs) # Initialize the Debugger self.debugger = Debugger(self.log, self.debugpy_stream, self._publish_debug_event, self.debug_shell_socket, self.session) # Initialize the InteractiveShell subclass self.shell = self.shell_class.instance(parent=self, profile_dir = self.profile_dir, user_module = self.user_module, user_ns = self.user_ns, kernel = self, compiler_class = XCachingCompiler, ) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket self.shell.displayhook.topic = self._topic('execute_result') self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket self.comm_manager = CommManager(parent=self, kernel=self) self.shell.configurables.append(self.comm_manager) comm_msg_types = [ 'comm_open', 'comm_msg', 'comm_close' ] for msg_type in comm_msg_types: self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type) if _use_appnope() and self._darwin_app_nap: # Disable app-nap as the kernel is not a gui but can have guis import appnope appnope.nope() help_links = List([ { 'text': "Python Reference", 'url': "https://docs.python.org/%i.%i" % sys.version_info[:2], }, { 'text': "IPython Reference", 'url': "https://ipython.org/documentation.html", }, { 'text': "NumPy Reference", 'url': "https://docs.scipy.org/doc/numpy/reference/", }, { 'text': "SciPy Reference", 'url': "https://docs.scipy.org/doc/scipy/reference/", }, { 'text': "Matplotlib Reference", 'url': "https://matplotlib.org/contents.html", }, { 'text': "SymPy Reference", 'url': "http://docs.sympy.org/latest/index.html", }, { 'text': "pandas Reference", 'url': "https://pandas.pydata.org/pandas-docs/stable/", }, ]).tag(config=True) # Kernel info fields implementation = 'ipython' implementation_version = release.version language_info = { 'name': 'python', 'version': sys.version.split()[0], 'mimetype': 'text/x-python', 'codemirror_mode': { 'name': 'ipython', 'version': sys.version_info[0] }, 'pygments_lexer': 'ipython%d' % 3, 'nbconvert_exporter': 'python', 'file_extension': '.py' } @gen.coroutine def dispatch_debugpy(self, msg): # The first frame is the socket id, we can drop it frame = msg[1].bytes.decode('utf-8') self.log.debug("Debugpy received: %s", frame) self.debugger.tcp_client.receive_dap_frame(frame) @property def banner(self): return self.shell.banner def start(self): self.shell.exit_now = False self.debugpy_stream.on_recv(self.dispatch_debugpy, copy=False) super(IPythonKernel, self).start() def set_parent(self, ident, parent, channel='shell'): """Overridden from parent to tell the display hook and output streams about the parent message. """ super(IPythonKernel, self).set_parent(ident, parent, channel) if channel == 'shell': self.shell.set_parent(parent) def init_metadata(self, parent): """Initialize metadata. Run at the beginning of each execution request. """ md = super(IPythonKernel, self).init_metadata(parent) # FIXME: remove deprecated ipyparallel-specific code # This is required for ipyparallel < 5.0 md.update({ 'dependencies_met' : True, 'engine' : self.ident, }) return md def finish_metadata(self, parent, metadata, reply_content): """Finish populating metadata. Run after completing an execution request. """ # FIXME: remove deprecated ipyparallel-specific code # This is required by ipyparallel < 5.0 metadata['status'] = reply_content['status'] if reply_content['status'] == 'error' and reply_content['ename'] == 'UnmetDependency': metadata['dependencies_met'] = False return metadata def _forward_input(self, allow_stdin=False): """Forward raw_input and getpass to the current frontend. via input_request """ self._allow_stdin = allow_stdin self._sys_raw_input = builtins.input builtins.input = self.raw_input self._save_getpass = getpass.getpass getpass.getpass = self.getpass def _restore_input(self): """Restore raw_input, getpass""" builtins.input = self._sys_raw_input getpass.getpass = self._save_getpass @property def execution_count(self): return self.shell.execution_count @execution_count.setter def execution_count(self, value): # Ignore the incrementing done by KernelBase, in favour of our shell's # execution counter. pass @contextmanager def _cancel_on_sigint(self, future): """ContextManager for capturing SIGINT and cancelling a future SIGINT raises in the event loop when running async code, but we want it to halt a coroutine. Ideally, it would raise KeyboardInterrupt, but this turns it into a CancelledError. At least it gets a decent traceback to the user. """ sigint_future = asyncio.Future() # whichever future finishes first, # cancel the other one def cancel_unless_done(f, _ignored): if f.cancelled() or f.done(): return f.cancel() # when sigint finishes, # abort the coroutine with CancelledError sigint_future.add_done_callback( partial(cancel_unless_done, future) ) # when the main future finishes, # stop watching for SIGINT events future.add_done_callback( partial(cancel_unless_done, sigint_future) ) def handle_sigint(*args): def set_sigint_result(): if sigint_future.cancelled() or sigint_future.done(): return sigint_future.set_result(1) # use add_callback for thread safety self.io_loop.add_callback(set_sigint_result) # set the custom sigint hander during this context save_sigint = signal.signal(signal.SIGINT, handle_sigint) try: yield finally: # restore the previous sigint handler signal.signal(signal.SIGINT, save_sigint) @gen.coroutine def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): shell = self.shell # we'll need this a lot here self._forward_input(allow_stdin) reply_content = {} if hasattr(shell, 'run_cell_async') and hasattr(shell, 'should_run_async'): run_cell = shell.run_cell_async should_run_async = shell.should_run_async else: should_run_async = lambda cell: False # older IPython, # use blocking run_cell and wrap it in coroutine @gen.coroutine def run_cell(*args, **kwargs): return shell.run_cell(*args, **kwargs) try: # default case: runner is asyncio and asyncio is already running # TODO: this should check every case for "are we inside the runner", # not just asyncio preprocessing_exc_tuple = None try: transformed_cell = self.shell.transform_cell(code) except Exception: transformed_cell = code preprocessing_exc_tuple = sys.exc_info() if ( _asyncio_runner and shell.loop_runner is _asyncio_runner and asyncio.get_event_loop().is_running() and should_run_async(code, transformed_cell=transformed_cell, preprocessing_exc_tuple=preprocessing_exc_tuple) ): coro = run_cell( code, store_history=store_history, silent=silent, transformed_cell=transformed_cell, preprocessing_exc_tuple=preprocessing_exc_tuple ) coro_future = asyncio.ensure_future(coro) with self._cancel_on_sigint(coro_future): res = None try: res = yield coro_future finally: shell.events.trigger('post_execute') if not silent: shell.events.trigger('post_run_cell', res) else: # runner isn't already running, # make synchronous call, # letting shell dispatch to loop runners res = shell.run_cell(code, store_history=store_history, silent=silent) finally: self._restore_input() if res.error_before_exec is not None: err = res.error_before_exec else: err = res.error_in_exec if res.success: reply_content['status'] = 'ok' else: reply_content['status'] = 'error' reply_content.update({ 'traceback': shell._last_traceback or [], 'ename': str(type(err).__name__), 'evalue': safe_unicode(err), }) # FIXME: deprecated piece for ipyparallel (remove in 5.0): e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute') reply_content['engine_info'] = e_info # Return the execution counter so clients can display prompts reply_content['execution_count'] = shell.execution_count - 1 if 'traceback' in reply_content: self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback'])) # At this point, we can tell whether the main code execution succeeded # or not. If it did, we proceed to evaluate user_expressions if reply_content['status'] == 'ok': reply_content['user_expressions'] = \ shell.user_expressions(user_expressions or {}) else: # If there was an error, don't even try to compute expressions reply_content['user_expressions'] = {} # Payloads should be retrieved regardless of outcome, so we can both # recover partial output (that could have been generated early in a # block, before an error) and always clear the payload system. reply_content['payload'] = shell.payload_manager.read_payload() # Be aggressive about clearing the payload because we don't want # it to sit in memory until the next execute_request comes in. shell.payload_manager.clear_payload() return reply_content def do_complete(self, code, cursor_pos): if _use_experimental_60_completion and self.use_experimental_completions: return self._experimental_do_complete(code, cursor_pos) # FIXME: IPython completers currently assume single line, # but completion messages give multi-line context # For now, extract line from cell, based on cursor_pos: if cursor_pos is None: cursor_pos = len(code) line, offset = line_at_cursor(code, cursor_pos) line_cursor = cursor_pos - offset txt, matches = self.shell.complete('', line, line_cursor) return {'matches' : matches, 'cursor_end' : cursor_pos, 'cursor_start' : cursor_pos - len(txt), 'metadata' : {}, 'status' : 'ok'} @gen.coroutine def do_debug_request(self, msg): return (yield self.debugger.process_request(msg)) def _experimental_do_complete(self, code, cursor_pos): """ Experimental completions from IPython, using Jedi. """ if cursor_pos is None: cursor_pos = len(code) with _provisionalcompleter(): raw_completions = self.shell.Completer.completions(code, cursor_pos) completions = list(_rectify_completions(code, raw_completions)) comps = [] for comp in completions: comps.append(dict( start=comp.start, end=comp.end, text=comp.text, type=comp.type, )) if completions: s = completions[0].start e = completions[0].end matches = [c.text for c in completions] else: s = cursor_pos e = cursor_pos matches = [] return {'matches': matches, 'cursor_end': e, 'cursor_start': s, 'metadata': {_EXPERIMENTAL_KEY_NAME: comps}, 'status': 'ok'} def do_inspect(self, code, cursor_pos, detail_level=0): name = token_at_cursor(code, cursor_pos) reply_content = {'status' : 'ok'} reply_content['data'] = {} reply_content['metadata'] = {} try: reply_content['data'].update( self.shell.object_inspect_mime( name, detail_level=detail_level ) ) if not self.shell.enable_html_pager: reply_content['data'].pop('text/html') reply_content['found'] = True except KeyError: reply_content['found'] = False return reply_content def do_history(self, hist_access_type, output, raw, session=0, start=0, stop=None, n=None, pattern=None, unique=False): if hist_access_type == 'tail': hist = self.shell.history_manager.get_tail(n, raw=raw, output=output, include_latest=True) elif hist_access_type == 'range': hist = self.shell.history_manager.get_range(session, start, stop, raw=raw, output=output) elif hist_access_type == 'search': hist = self.shell.history_manager.search( pattern, raw=raw, output=output, n=n, unique=unique) else: hist = [] return { 'status': 'ok', 'history' : list(hist), } def do_shutdown(self, restart): self.shell.exit_now = True return dict(status='ok', restart=restart) def do_is_complete(self, code): transformer_manager = getattr(self.shell, 'input_transformer_manager', None) if transformer_manager is None: # input_splitter attribute is deprecated transformer_manager = self.shell.input_splitter status, indent_spaces = transformer_manager.check_complete(code) r = {'status': status} if status == 'incomplete': r['indent'] = ' ' * indent_spaces return r def do_apply(self, content, bufs, msg_id, reply_metadata): from .serialize import serialize_object, unpack_apply_message shell = self.shell try: working = shell.user_ns prefix = "_"+str(msg_id).replace("-","")+"_" f,args,kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix+"f" argname = prefix+"args" kwargname = prefix+"kwargs" resultname = prefix+"result" ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object(result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except BaseException as e: # invoke IPython traceback formatting shell.showtraceback() reply_content = { 'traceback': shell._last_traceback or [], 'ename': str(type(e).__name__), 'evalue': safe_unicode(e), } # FIXME: deprecated piece for ipyparallel (remove in 5.0): e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info self.send_response(self.iopub_socket, 'error', reply_content, ident=self._topic('error'), channel='shell') self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] reply_content['status'] = 'error' else: reply_content = {'status' : 'ok'} return reply_content, result_buf def do_clear(self): self.shell.reset(False) return dict(status='ok')
class TerminalInteractiveShell(InteractiveShell): space_for_menu = Integer( 6, help='Number of line at the bottom of the screen ' 'to reserve for the completion menu').tag(config=True) def _space_for_menu_changed(self, old, new): self._update_layout() pt_cli = None debugger_history = None _pt_app = None simple_prompt = Bool( _use_simple_prompt, help= """Use `raw_input` for the REPL, without completion, multiline input, and prompt colors. Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are: IPython own testing machinery, and emacs inferior-shell integration through elpy. This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT` environment variable is set, or the current terminal is not a tty.""" ).tag(config=True) @property def debugger_cls(self): return Pdb if self.simple_prompt else TerminalPdb confirm_exit = Bool( True, help=""" Set to confirm when you try to exit IPython with an EOF (Control-D in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', you can force a direct exit without any confirmation.""", ).tag(config=True) editing_mode = Unicode( 'emacs', help="Shortcut style to use at the prompt. 'vi' or 'emacs'.", ).tag(config=True) mouse_support = Bool( False, help="Enable mouse support in the prompt").tag(config=True) highlighting_style = Union( [Unicode('legacy'), Type(klass=Style)], help="""The name or class of a Pygments style to use for syntax highlighting: \n %s""" % ', '.join(get_all_styles())).tag(config=True) @observe('highlighting_style') @observe('colors') def _highlighting_style_changed(self, change): self.refresh_style() def refresh_style(self): self._style = self._make_style_from_name_or_cls( self.highlighting_style) highlighting_style_overrides = Dict( help="Override highlighting format for specific tokens").tag( config=True) true_color = Bool( False, help=("Use 24bit colors instead of 256 colors in prompt highlighting. " "If your terminal supports true color, the following command " "should print 'TRUECOLOR' in orange: " "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")).tag( config=True) editor = Unicode( get_default_editor(), help="Set the editor used by IPython (default to $EDITOR/vi/notepad)." ).tag(config=True) prompts_class = Type( Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag( config=True) prompts = Instance(Prompts) @default('prompts') def _prompts_default(self): return self.prompts_class(self) @observe('prompts') def _(self, change): self._update_layout() @default('displayhook_class') def _displayhook_class_default(self): return RichPromptDisplayHook term_title = Bool( True, help="Automatically set the terminal title").tag(config=True) display_completions = Enum( ('column', 'multicolumn', 'readlinelike'), help= ("Options for displaying tab completions, 'column', 'multicolumn', and " "'readlinelike'. These options are for `prompt_toolkit`, see " "`prompt_toolkit` documentation for more information."), default_value='multicolumn').tag(config=True) highlight_matching_brackets = Bool( True, help="Highlight matching brackets.", ).tag(config=True) extra_open_editor_shortcuts = Bool( False, help= "Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. " "This is in addition to the F2 binding, which is always enabled.").tag( config=True) handle_return = Any( None, help="Provide an alternative handler to be called when the user presses " "Return. This is an advanced option intended for debugging, which " "may be changed or removed in later releases.").tag(config=True) @observe('term_title') def init_term_title(self, change=None): # Enable or disable the terminal title. if self.term_title: toggle_set_term_title(True) set_term_title('IPython: ' + abbrev_cwd()) else: toggle_set_term_title(False) def init_display_formatter(self): super(TerminalInteractiveShell, self).init_display_formatter() # terminal only supports plain text self.display_formatter.active_types = ['text/plain'] # disable `_ipython_display_` self.display_formatter.ipython_display_formatter.enabled = False def init_prompt_toolkit_cli(self): if self.simple_prompt: # Fall back to plain non-interactive output for tests. # This is very limited, and only accepts a single line. def prompt(): return cast_unicode_py2( input('In [%d]: ' % self.execution_count)) self.prompt_for_code = prompt return # Set up keyboard shortcuts kbmanager = KeyBindingManager.for_prompt( enable_open_in_editor=self.extra_open_editor_shortcuts, ) register_ipython_shortcuts(kbmanager.registry, self) # Pre-populate history from IPython's history database history = InMemoryHistory() last_cell = u"" for __, ___, cell in self.history_manager.get_tail( self.history_load_length, include_latest=True): # Ignore blank lines and consecutive duplicates cell = cell.rstrip() if cell and (cell != last_cell): history.append(cell) last_cell = cell self._style = self._make_style_from_name_or_cls( self.highlighting_style) self.style = DynamicStyle(lambda: self._style) editing_mode = getattr(EditingMode, self.editing_mode.upper()) def patch_stdout(**kwargs): return self.pt_cli.patch_stdout_context(**kwargs) self._pt_app = create_prompt_application( editing_mode=editing_mode, key_bindings_registry=kbmanager.registry, history=history, completer=IPythonPTCompleter(shell=self, patch_stdout=patch_stdout), enable_history_search=True, style=self.style, mouse_support=self.mouse_support, **self._layout_options()) self._eventloop = create_eventloop(self.inputhook) self.pt_cli = CommandLineInterface( self._pt_app, eventloop=self._eventloop, output=create_output(true_color=self.true_color)) def _make_style_from_name_or_cls(self, name_or_cls): """ Small wrapper that make an IPython compatible style from a style name We need that to add style for prompt ... etc. """ style_overrides = {} if name_or_cls == 'legacy': legacy = self.colors.lower() if legacy == 'linux': style_cls = get_style_by_name('monokai') style_overrides = _style_overrides_linux elif legacy == 'lightbg': style_overrides = _style_overrides_light_bg style_cls = get_style_by_name('pastie') elif legacy == 'neutral': # The default theme needs to be visible on both a dark background # and a light background, because we can't tell what the terminal # looks like. These tweaks to the default theme help with that. style_cls = get_style_by_name('default') style_overrides.update({ Token.Number: '#007700', Token.Operator: 'noinherit', Token.String: '#BB6622', Token.Name.Function: '#2080D0', Token.Name.Class: 'bold #2080D0', Token.Name.Namespace: 'bold #2080D0', Token.Prompt: '#009900', Token.PromptNum: '#00ff00 bold', Token.OutPrompt: '#990000', Token.OutPromptNum: '#ff0000 bold', }) # Hack: Due to limited color support on the Windows console # the prompt colors will be wrong without this if os.name == 'nt': style_overrides.update({ Token.Prompt: '#ansidarkgreen', Token.PromptNum: '#ansigreen bold', Token.OutPrompt: '#ansidarkred', Token.OutPromptNum: '#ansired bold', }) elif legacy == 'nocolor': style_cls = _NoStyle style_overrides = {} else: raise ValueError('Got unknown colors: ', legacy) else: if isinstance(name_or_cls, str): style_cls = get_style_by_name(name_or_cls) else: style_cls = name_or_cls style_overrides = { Token.Prompt: '#009900', Token.PromptNum: '#00ff00 bold', Token.OutPrompt: '#990000', Token.OutPromptNum: '#ff0000 bold', } style_overrides.update(self.highlighting_style_overrides) style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls, style_dict=style_overrides) return style def _layout_options(self): """ Return the current layout option for the current Terminal InteractiveShell """ return { 'lexer': IPythonPTLexer(), 'reserve_space_for_menu': self.space_for_menu, 'get_prompt_tokens': self.prompts.in_prompt_tokens, 'get_continuation_tokens': self.prompts.continuation_prompt_tokens, 'multiline': True, 'display_completions_in_columns': (self.display_completions == 'multicolumn'), # Highlight matching brackets, but only when this setting is # enabled, and only when the DEFAULT_BUFFER has the focus. 'extra_input_processors': [ ConditionalProcessor( processor=HighlightMatchingBracketProcessor( chars='[](){}'), filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() & Condition(lambda cli: self.highlight_matching_brackets)) ], } def _update_layout(self): """ Ask for a re computation of the application layout, if for example , some configuration options have changed. """ if self._pt_app: self._pt_app.layout = create_prompt_layout( **self._layout_options()) def prompt_for_code(self): document = self.pt_cli.run(pre_run=self.pre_prompt, reset_current_buffer=True) return document.text def enable_win_unicode_console(self): if sys.version_info >= (3, 6): # Since PEP 528, Python uses the unicode APIs for the Windows # console by default, so WUC shouldn't be needed. return import win_unicode_console win_unicode_console.enable() def init_io(self): if sys.platform not in {'win32', 'cli'}: return self.enable_win_unicode_console() import colorama colorama.init() # For some reason we make these wrappers around stdout/stderr. # For now, we need to reset them so all output gets coloured. # https://github.com/ipython/ipython/issues/8669 # io.std* are deprecated, but don't show our own deprecation warnings # during initialization of the deprecated API. with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) io.stdout = io.IOStream(sys.stdout) io.stderr = io.IOStream(sys.stderr) def init_magics(self): super(TerminalInteractiveShell, self).init_magics() self.register_magics(TerminalMagics) def init_alias(self): # The parent class defines aliases that can be safely used with any # frontend. super(TerminalInteractiveShell, self).init_alias() # Now define aliases that only make sense on the terminal, because they # need direct access to the console in a way that we can't emulate in # GUI or web frontend if os.name == 'posix': for cmd in ['clear', 'more', 'less', 'man']: self.alias_manager.soft_define_alias(cmd, cmd) def __init__(self, *args, **kwargs): super(TerminalInteractiveShell, self).__init__(*args, **kwargs) self.init_prompt_toolkit_cli() self.init_term_title() self.keep_running = True self.debugger_history = InMemoryHistory() def ask_exit(self): self.keep_running = False rl_next_input = None def pre_prompt(self): if self.rl_next_input: # We can't set the buffer here, because it will be reset just after # this. Adding a callable to pre_run_callables does what we need # after the buffer is reset. s = cast_unicode_py2(self.rl_next_input) def set_doc(): self.pt_cli.application.buffer.document = Document(s) if hasattr(self.pt_cli, 'pre_run_callables'): self.pt_cli.pre_run_callables.append(set_doc) else: # Older version of prompt_toolkit; it's OK to set the document # directly here. set_doc() self.rl_next_input = None def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED): if display_banner is not DISPLAY_BANNER_DEPRECATED: warn( 'interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2) self.keep_running = True while self.keep_running: print(self.separate_in, end='') try: code = self.prompt_for_code() except EOFError: if (not self.confirm_exit) \ or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'): self.ask_exit() else: if code: self.run_cell(code, store_history=True) def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED): # An extra layer of protection in case someone mashing Ctrl-C breaks # out of our internal code. if display_banner is not DISPLAY_BANNER_DEPRECATED: warn( 'mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2) while True: try: self.interact() break except KeyboardInterrupt as e: print("\n%s escaped interact()\n" % type(e).__name__) finally: # An interrupt during the eventloop will mess up the # internal state of the prompt_toolkit library. # Stopping the eventloop fixes this, see # https://github.com/ipython/ipython/pull/9867 if hasattr(self, '_eventloop'): self._eventloop.stop() _inputhook = None def inputhook(self, context): if self._inputhook is not None: self._inputhook(context) active_eventloop = None def enable_gui(self, gui=None): if gui: self.active_eventloop, self._inputhook =\ get_inputhook_name_and_func(gui) else: self.active_eventloop = self._inputhook = None # Run !system commands directly, not through pipes, so terminal programs # work correctly. system = InteractiveShell.system_raw def auto_rewrite_input(self, cmd): """Overridden from the parent class to use fancy rewriting prompt""" if not self.show_rewritten_input: return tokens = self.prompts.rewrite_prompt_tokens() if self.pt_cli: self.pt_cli.print_tokens(tokens) print(cmd) else: prompt = ''.join(s for t, s in tokens) print(prompt, cmd, sep='') _prompts_before = None def switch_doctest_mode(self, mode): """Switch prompts to classic for %doctest_mode""" if mode: self._prompts_before = self.prompts self.prompts = ClassicPrompts(self) elif self._prompts_before: self.prompts = self._prompts_before self._prompts_before = None self._update_layout()
class Service(LoggingConfigurable): """An object wrapping a service specification for Hub API consumers. A service has inputs: - name: str the name of the service - admin: bool(false) whether the service should have administrative privileges - url: str (None) The URL where the service is/should be. If specified, the service will be added to the proxy at /services/:name If a service is to be managed by the Hub, it has a few extra options: - command: (str/Popen list) Command for JupyterHub to spawn the service. Only use this if the service should be a subprocess. If command is not specified, it is assumed to be managed by a - environment: dict Additional environment variables for the service. - user: str The name of a system user to become. If unspecified, run as the same user as the Hub. """ # inputs: name = Unicode( help="""The name of the service. If the service has an http endpoint, it """ ).tag(input=True) admin = Bool(False, help="Does the service need admin-access to the Hub API?" ).tag(input=True) url = Unicode( help="""URL of the service. Only specify if the service runs an HTTP(s) endpoint that. If managed, will be passed as JUPYTERHUB_SERVICE_URL env. """ ).tag(input=True) api_token = Unicode( help="""The API token to use for the service. If unspecified, an API token will be generated for managed services. """ ).tag(input=True) # Managed service API: @property def managed(self): """Am I managed by the Hub?""" return bool(self.command) @property def kind(self): """The name of the kind of service as a string - 'managed' for managed services - 'external' for external services """ return 'managed' if self.managed else 'external' command = Command(minlen=0, help="Command to spawn this service, if managed." ).tag(input=True) cwd = Unicode( help="""The working directory in which to run the service.""" ).tag(input=True) environment = Dict( help="""Environment variables to pass to the service. Only used if the Hub is spawning the service. """ ).tag(input=True) user = Unicode(getuser(), help="""The user to become when launching the service. If unspecified, run the service as the same user as the Hub. """ ).tag(input=True) domain = Unicode() host = Unicode() hub = Any() proc = Any() # handles on globals: proxy = Any() base_url = Unicode() db = Any() orm = Any() oauth_provider = Any() oauth_client_id = Unicode( help="""OAuth client ID for this service. You shouldn't generally need to change this. Default: `service-<name>` """ ).tag(input=True) @default('oauth_client_id') def _default_client_id(self): return 'service-%s' % self.name @property def server(self): return self.orm.server @property def prefix(self): return url_path_join(self.base_url, 'services', self.name + '/') @property def proxy_path(self): if not self.server: return '' if self.domain: return url_path_join('/' + self.domain, self.server.base_url) else: return self.server.base_url def __repr__(self): return "<{cls}(name={name}{managed})>".format( cls=self.__class__.__name__, name=self.name, managed=' managed' if self.managed else '', ) def start(self): """Start a managed service""" if not self.managed: raise RuntimeError("Cannot start unmanaged service %s" % self) self.log.info("Starting service %r: %r", self.name, self.command) env = {} env.update(self.environment) env['JUPYTERHUB_SERVICE_NAME'] = self.name env['JUPYTERHUB_API_TOKEN'] = self.api_token env['JUPYTERHUB_API_URL'] = self.hub_api_url env['JUPYTERHUB_BASE_URL'] = self.base_url if self.url: env['JUPYTERHUB_SERVICE_URL'] = self.url env['JUPYTERHUB_SERVICE_PREFIX'] = self.server.base_url self.spawner = _ServiceSpawner( cmd=self.command, environment=env, api_token=self.api_token, oauth_client_id=self.oauth_client_id, cwd=self.cwd, hub=self.hub, user=_MockUser( name=self.user, service=self, server=self.orm.server, host=self.host, ), ) self.spawner.start() self.proc = self.spawner.proc self.spawner.add_poll_callback(self._proc_stopped) self.spawner.start_polling() def _proc_stopped(self): """Called when the service process unexpectedly exits""" self.log.error("Service %s exited with status %i", self.name, self.proc.returncode) self.start() def stop(self): """Stop a managed service""" if not self.managed: raise RuntimeError("Cannot start unmanaged service %s" % self) self.spawner.stop_polling() return self.spawner.stop()
class PapayaWidget(HBox): """A widget class that displays a papaya viewer (NlPapayaViewer) and config widget (PapayaConfigWidget) side by side.""" current_config = Any() current_colorbar = Any() def __init__(self, *args, **kwargs): """ Parameters ---------- **kwargs: config_visible: str Depending on the value, config widget will be visible or hidden upon initialization. Possible values are "hidden" or "visible" """ super().__init__(*args, **kwargs) self._viewer = NlPapayaViewer(layout=Layout( width="70%", height="auto", border="1px solid black")) self._config = PapayaConfigWidget( self._viewer, layout=Layout(width="30%", height="auto", border="1px solid black"), ) self._config.layout.visibility = kwargs.get("config_visible", "hidden") self.current_config = None self.current_colorbar = None self.children = [self._viewer, self._config] def show_image_config(self, image, show=True): if show: self._config.layout.visibility = "visible" self.current_config = image self._config.set_image(image) else: if self.current_config is not None and image.id == self.current_config.id: self._config.layout.visibility = "hidden" self._config.set_image(None) def show_image_colorbar(self, image): self.current_colorbar = image self._viewer.show_image_colorbar(image) def can_add(self, images): return self._viewer.can_add(images) def add(self, images): self._viewer.add(images) self.current_colorbar = self._viewer.get_colorbar_image() def remove(self, images): self._viewer.remove(images) self.current_colorbar = self._viewer.get_colorbar_image() def set_images(self): self._viewer.set_images() def set_center(self, widget, image): self._viewer.set_center(widget, image) def reset_center(self): self._viewer.reset_center() def set_error(self, error): self._viewer.set_error(error) def reset(self): self._viewer.reset() self._config.reset() def get_hex_for_lut(self, lut): return self._viewer.get_hex_for_lut(lut)
class Authenticator(LoggingConfigurable): """Base class for implementing an authentication provider for JupyterHub""" db = Any() admin_users = Set( help=""" Set of users that will have admin rights on this JupyterHub. Admin users have extra privileges: - Use the admin panel to see list of users logged in - Add / remove users in some authenticators - Restart / halt the hub - Start / stop users' single-user servers - Can access each individual users' single-user server (if configured) Admin access should be treated the same way root access is. Defaults to an empty set, in which case no user has admin access. """ ).tag(config=True) whitelist = Set( help=""" Whitelist of usernames that are allowed to log in. Use this with supported authenticators to restrict which users can log in. This is an additional whitelist that further restricts users, beyond whatever restrictions the authenticator has in place. If empty, does not perform any additional restriction. """ ).tag(config=True) @observe('whitelist') def _check_whitelist(self, change): short_names = [name for name in change['new'] if len(name) <= 1] if short_names: sorted_names = sorted(short_names) single = ''.join(sorted_names) string_set_typo = "set('%s')" % single self.log.warning("whitelist contains single-character names: %s; did you mean set([%r]) instead of %s?", sorted_names[:8], single, string_set_typo, ) custom_html = Unicode( help=""" HTML form to be overridden by authenticators if they want a custom authentication form. Defaults to an empty string, which shows the default username/password form. """ ) login_service = Unicode( help=""" Name of the login service that this authenticator is providing using to authenticate users. Example: GitHub, MediaWiki, Google, etc. Setting this value replaces the login form with a "Login with <login_service>" button. Any authenticator that redirects to an external service (e.g. using OAuth) should set this. """ ) username_pattern = Unicode( help=""" Regular expression pattern that all valid usernames must match. If a username does not match the pattern specified here, authentication will not be attempted. If not set, allow any username. """ ).tag(config=True) @observe('username_pattern') def _username_pattern_changed(self, change): if not change['new']: self.username_regex = None self.username_regex = re.compile(change['new']) username_regex = Any( help=""" Compiled regex kept in sync with `username_pattern` """ ) def validate_username(self, username): """Validate a normalized username Return True if username is valid, False otherwise. """ if not self.username_regex: return True return bool(self.username_regex.match(username)) username_map = Dict( help="""Dictionary mapping authenticator usernames to JupyterHub users. Primarily used to normalize OAuth user names to local users. """ ).tag(config=True) delete_invalid_users = Bool(False, help="""Delete any users from the database that do not pass validation When JupyterHub starts, `.add_user` will be called on each user in the database to verify that all users are still valid. If `delete_invalid_users` is True, any users that do not pass validation will be deleted from the database. Use this if users might be deleted from an external system, such as local user accounts. If False (default), invalid users remain in the Hub's database and a warning will be issued. This is the default to avoid data loss due to config changes. """ ) def normalize_username(self, username): """Normalize the given username and return it Override in subclasses if usernames need different normalization rules. The default attempts to lowercase the username and apply `username_map` if it is set. """ username = username.lower() username = self.username_map.get(username, username) return username def check_whitelist(self, username): """Check if a username is allowed to authenticate based on whitelist configuration Return True if username is allowed, False otherwise. No whitelist means any username is allowed. Names are normalized *before* being checked against the whitelist. """ if not self.whitelist: # No whitelist means any name is allowed return True return username in self.whitelist @gen.coroutine def get_authenticated_user(self, handler, data): """Authenticate the user who is attempting to log in Returns normalized username if successful, None otherwise. This calls `authenticate`, which should be overridden in subclasses, normalizes the username if any normalization should be done, and then validates the name in the whitelist. This is the outer API for authenticating a user. Subclasses should not need to override this method. The various stages can be overridden separately: - `authenticate` turns formdata into a username - `normalize_username` normalizes the username - `check_whitelist` checks against the user whitelist """ username = yield self.authenticate(handler, data) if username is None: return username = self.normalize_username(username) if not self.validate_username(username): self.log.warning("Disallowing invalid username %r.", username) return whitelist_pass = yield gen.maybe_future(self.check_whitelist(username)) if whitelist_pass: return username else: self.log.warning("User %r not in whitelist.", username) return @gen.coroutine def authenticate(self, handler, data): """Authenticate a user with login form data This must be a tornado gen.coroutine. It must return the username on successful authentication, and return None on failed authentication. Checking the whitelist is handled separately by the caller. Args: handler (tornado.web.RequestHandler): the current request handler data (dict): The formdata of the login form. The default form has 'username' and 'password' fields. Returns: username (str or None): The username of the authenticated user, or None if Authentication failed """ def pre_spawn_start(self, user, spawner): """Hook called before spawning a user's server Can be used to do auth-related startup, e.g. opening PAM sessions. """ def post_spawn_stop(self, user, spawner): """Hook called after stopping a user container Can be used to do auth-related cleanup, e.g. closing PAM sessions. """ def add_user(self, user): """Hook called when a user is added to JupyterHub This is called: - When a user first authenticates - When the hub restarts, for all users. This method may be a coroutine. By default, this just adds the user to the whitelist. Subclasses may do more extensive things, such as adding actual unix users, but they should call super to ensure the whitelist is updated. Note that this should be idempotent, since it is called whenever the hub restarts for all users. Args: user (User): The User wrapper object """ if not self.validate_username(user.name): raise ValueError("Invalid username: %s" % user.name) if self.whitelist: self.whitelist.add(user.name) def delete_user(self, user): """Hook called when a user is deleted Removes the user from the whitelist. Subclasses should call super to ensure the whitelist is updated. Args: user (User): The User wrapper object """ self.whitelist.discard(user.name) auto_login = Bool(False, config=True, help="""Automatically begin the login process rather than starting with a "Login with..." link at `/hub/login` To work, `.login_url()` must give a URL other than the default `/hub/login`, such as an oauth handler or another automatic login handler, registered with `.get_handlers()`. .. versionadded:: 0.8 """ ) def login_url(self, base_url): """Override this when registering a custom login handler Generally used by authenticators that do not use simple form-based authentication. The subclass overriding this is responsible for making sure there is a handler available to handle the URL returned from this method, using the `get_handlers` method. Args: base_url (str): the base URL of the Hub (e.g. /hub/) Returns: str: The login URL, e.g. '/hub/login' """ return url_path_join(base_url, 'login') def logout_url(self, base_url): """Override when registering a custom logout handler The subclass overriding this is responsible for making sure there is a handler available to handle the URL returned from this method, using the `get_handlers` method. Args: base_url (str): the base URL of the Hub (e.g. /hub/) Returns: str: The logout URL, e.g. '/hub/logout' """ return url_path_join(base_url, 'logout') def get_handlers(self, app): """Return any custom handlers the authenticator needs to register Used in conjugation with `login_url` and `logout_url`. Args: app (JupyterHub Application): the application object, in case it needs to be accessed for info. Returns: handlers (list): list of ``('/url', Handler)`` tuples passed to tornado. The Hub prefix is added to any URLs. """ return [ ('/login', LoginHandler), ]
class TerminalInteractiveShell(InteractiveShell): mime_renderers = Dict().tag(config=True) space_for_menu = Integer( 6, help='Number of line at the bottom of the screen ' 'to reserve for the tab completion menu, ' 'search history, ...etc, the height of ' 'these menus will at most this value. ' 'Increase it is you prefer long and skinny ' 'menus, decrease for short and wide.').tag(config=True) pt_app = None debugger_history = None simple_prompt = Bool( _use_simple_prompt, help= """Use `raw_input` for the REPL, without completion and prompt colors. Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are: IPython own testing machinery, and emacs inferior-shell integration through elpy. This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT` environment variable is set, or the current terminal is not a tty.""" ).tag(config=True) @property def debugger_cls(self): return Pdb if self.simple_prompt else TerminalPdb confirm_exit = Bool( True, help=""" Set to confirm when you try to exit IPython with an EOF (Control-D in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', you can force a direct exit without any confirmation.""", ).tag(config=True) editing_mode = Unicode( 'emacs', help="Shortcut style to use at the prompt. 'vi' or 'emacs'.", ).tag(config=True) autoformatter = Unicode( None, help= "Autoformatter to reformat Terminal code. Can be `'black'` or `None`", allow_none=True).tag(config=True) mouse_support = Bool( False, help= "Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)" ).tag(config=True) # We don't load the list of styles for the help string, because loading # Pygments plugins takes time and can cause unexpected errors. highlighting_style = Union( [Unicode('legacy'), Type(klass=Style)], help="""The name or class of a Pygments style to use for syntax highlighting. To see available styles, run `pygmentize -L styles`.""" ).tag(config=True) @validate('editing_mode') def _validate_editing_mode(self, proposal): if proposal['value'].lower() == 'vim': proposal['value'] = 'vi' elif proposal['value'].lower() == 'default': proposal['value'] = 'emacs' if hasattr(EditingMode, proposal['value'].upper()): return proposal['value'].lower() return self.editing_mode @observe('editing_mode') def _editing_mode(self, change): if self.pt_app: self.pt_app.editing_mode = getattr(EditingMode, change.new.upper()) @observe('autoformatter') def _autoformatter_changed(self, change): formatter = change.new if formatter is None: self.reformat_handler = lambda x: x elif formatter == 'black': self.reformat_handler = black_reformat_handler else: raise ValueError @observe('highlighting_style') @observe('colors') def _highlighting_style_changed(self, change): self.refresh_style() def refresh_style(self): self._style = self._make_style_from_name_or_cls( self.highlighting_style) highlighting_style_overrides = Dict( help="Override highlighting format for specific tokens").tag( config=True) true_color = Bool( False, help=("Use 24bit colors instead of 256 colors in prompt highlighting. " "If your terminal supports true color, the following command " "should print 'TRUECOLOR' in orange: " "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")).tag( config=True) editor = Unicode( get_default_editor(), help="Set the editor used by IPython (default to $EDITOR/vi/notepad)." ).tag(config=True) prompts_class = Type( Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag( config=True) prompts = Instance(Prompts) @default('prompts') def _prompts_default(self): return self.prompts_class(self) # @observe('prompts') # def _(self, change): # self._update_layout() @default('displayhook_class') def _displayhook_class_default(self): return RichPromptDisplayHook term_title = Bool( True, help="Automatically set the terminal title").tag(config=True) term_title_format = Unicode( "IPython: {cwd}", help= "Customize the terminal title format. This is a python format string. " + "Available substitutions are: {cwd}.").tag(config=True) display_completions = Enum( ('column', 'multicolumn', 'readlinelike'), help= ("Options for displaying tab completions, 'column', 'multicolumn', and " "'readlinelike'. These options are for `prompt_toolkit`, see " "`prompt_toolkit` documentation for more information."), default_value='multicolumn').tag(config=True) highlight_matching_brackets = Bool( True, help="Highlight matching brackets.", ).tag(config=True) extra_open_editor_shortcuts = Bool( False, help= "Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. " "This is in addition to the F2 binding, which is always enabled.").tag( config=True) handle_return = Any( None, help="Provide an alternative handler to be called when the user presses " "Return. This is an advanced option intended for debugging, which " "may be changed or removed in later releases.").tag(config=True) enable_history_search = Bool( True, help="Allows to enable/disable the prompt toolkit history search").tag( config=True) prompt_includes_vi_mode = Bool( True, help="Display the current vi mode (when using vi editing mode).").tag( config=True) @observe('term_title') def init_term_title(self, change=None): # Enable or disable the terminal title. if self.term_title: toggle_set_term_title(True) set_term_title(self.term_title_format.format(cwd=abbrev_cwd())) else: toggle_set_term_title(False) def restore_term_title(self): if self.term_title: restore_term_title() def init_display_formatter(self): super(TerminalInteractiveShell, self).init_display_formatter() # terminal only supports plain text self.display_formatter.active_types = ['text/plain'] # disable `_ipython_display_` self.display_formatter.ipython_display_formatter.enabled = False def init_prompt_toolkit_cli(self): if self.simple_prompt: # Fall back to plain non-interactive output for tests. # This is very limited. def prompt(): prompt_text = "".join(x[1] for x in self.prompts.in_prompt_tokens()) lines = [input(prompt_text)] prompt_continuation = "".join( x[1] for x in self.prompts.continuation_prompt_tokens()) while self.check_complete('\n'.join(lines))[0] == 'incomplete': lines.append(input(prompt_continuation)) return '\n'.join(lines) self.prompt_for_code = prompt return # Set up keyboard shortcuts key_bindings = create_ipython_shortcuts(self) # Pre-populate history from IPython's history database history = InMemoryHistory() last_cell = u"" for __, ___, cell in self.history_manager.get_tail( self.history_load_length, include_latest=True): # Ignore blank lines and consecutive duplicates cell = cell.rstrip() if cell and (cell != last_cell): history.append_string(cell) last_cell = cell self._style = self._make_style_from_name_or_cls( self.highlighting_style) self.style = DynamicStyle(lambda: self._style) editing_mode = getattr(EditingMode, self.editing_mode.upper()) self.pt_loop = asyncio.new_event_loop() self.pt_app = PromptSession( editing_mode=editing_mode, key_bindings=key_bindings, history=history, completer=IPythonPTCompleter(shell=self), enable_history_search=self.enable_history_search, style=self.style, include_default_pygments_style=False, mouse_support=self.mouse_support, enable_open_in_editor=self.extra_open_editor_shortcuts, color_depth=self.color_depth, tempfile_suffix=".py", **self._extra_prompt_options()) def _make_style_from_name_or_cls(self, name_or_cls): """ Small wrapper that make an IPython compatible style from a style name We need that to add style for prompt ... etc. """ style_overrides = {} if name_or_cls == 'legacy': legacy = self.colors.lower() if legacy == 'linux': style_cls = get_style_by_name('monokai') style_overrides = _style_overrides_linux elif legacy == 'lightbg': style_overrides = _style_overrides_light_bg style_cls = get_style_by_name('pastie') elif legacy == 'neutral': # The default theme needs to be visible on both a dark background # and a light background, because we can't tell what the terminal # looks like. These tweaks to the default theme help with that. style_cls = get_style_by_name('default') style_overrides.update({ Token.Number: '#007700', Token.Operator: 'noinherit', Token.String: '#BB6622', Token.Name.Function: '#2080D0', Token.Name.Class: 'bold #2080D0', Token.Name.Namespace: 'bold #2080D0', Token.Name.Variable.Magic: '#ansiblue', Token.Prompt: '#009900', Token.PromptNum: '#ansibrightgreen bold', Token.OutPrompt: '#990000', Token.OutPromptNum: '#ansibrightred bold', }) # Hack: Due to limited color support on the Windows console # the prompt colors will be wrong without this if os.name == 'nt': style_overrides.update({ Token.Prompt: '#ansidarkgreen', Token.PromptNum: '#ansigreen bold', Token.OutPrompt: '#ansidarkred', Token.OutPromptNum: '#ansired bold', }) elif legacy == 'nocolor': style_cls = _NoStyle style_overrides = {} else: raise ValueError('Got unknown colors: ', legacy) else: if isinstance(name_or_cls, str): style_cls = get_style_by_name(name_or_cls) else: style_cls = name_or_cls style_overrides = { Token.Prompt: '#009900', Token.PromptNum: '#ansibrightgreen bold', Token.OutPrompt: '#990000', Token.OutPromptNum: '#ansibrightred bold', } style_overrides.update(self.highlighting_style_overrides) style = merge_styles([ style_from_pygments_cls(style_cls), style_from_pygments_dict(style_overrides), ]) return style @property def pt_complete_style(self): return { 'multicolumn': CompleteStyle.MULTI_COLUMN, 'column': CompleteStyle.COLUMN, 'readlinelike': CompleteStyle.READLINE_LIKE, }[self.display_completions] @property def color_depth(self): return (ColorDepth.TRUE_COLOR if self.true_color else None) def _extra_prompt_options(self): """ Return the current layout option for the current Terminal InteractiveShell """ def get_message(): return PygmentsTokens(self.prompts.in_prompt_tokens()) if self.editing_mode == 'emacs': # with emacs mode the prompt is (usually) static, so we call only # the function once. With VI mode it can toggle between [ins] and # [nor] so we can't precompute. # here I'm going to favor the default keybinding which almost # everybody uses to decrease CPU usage. # if we have issues with users with custom Prompts we can see how to # work around this. get_message = get_message() options = { 'complete_in_thread': False, 'lexer': IPythonPTLexer(), 'reserve_space_for_menu': self.space_for_menu, 'message': get_message, 'prompt_continuation': (lambda width, lineno, is_soft_wrap: PygmentsTokens( self.prompts.continuation_prompt_tokens(width))), 'multiline': True, 'complete_style': self.pt_complete_style, # Highlight matching brackets, but only when this setting is # enabled, and only when the DEFAULT_BUFFER has the focus. 'input_processors': [ ConditionalProcessor( processor=HighlightMatchingBracketProcessor( chars='[](){}'), filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() & Condition(lambda: self.highlight_matching_brackets)) ], } if not PTK3: options['inputhook'] = self.inputhook return options def prompt_for_code(self): if self.rl_next_input: default = self.rl_next_input self.rl_next_input = None else: default = '' # In order to make sure that asyncio code written in the # interactive shell doesn't interfere with the prompt, we run the # prompt in a different event loop. # If we don't do this, people could spawn coroutine with a # while/true inside which will freeze the prompt. try: old_loop = asyncio.get_event_loop() except RuntimeError: # This happens when the user used `asyncio.run()`. old_loop = None asyncio.set_event_loop(self.pt_loop) try: with patch_stdout(raw=True): text = self.pt_app.prompt(default=default, **self._extra_prompt_options()) finally: # Restore the original event loop. asyncio.set_event_loop(old_loop) return text def enable_win_unicode_console(self): # Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows # console by default, so WUC shouldn't be needed. from warnings import warn warn( "`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future", DeprecationWarning, stacklevel=2) def init_io(self): if sys.platform not in {'win32', 'cli'}: return import colorama colorama.init() # For some reason we make these wrappers around stdout/stderr. # For now, we need to reset them so all output gets coloured. # https://github.com/ipython/ipython/issues/8669 # io.std* are deprecated, but don't show our own deprecation warnings # during initialization of the deprecated API. with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) io.stdout = io.IOStream(sys.stdout) io.stderr = io.IOStream(sys.stderr) def init_magics(self): super(TerminalInteractiveShell, self).init_magics() self.register_magics(TerminalMagics) def init_alias(self): # The parent class defines aliases that can be safely used with any # frontend. super(TerminalInteractiveShell, self).init_alias() # Now define aliases that only make sense on the terminal, because they # need direct access to the console in a way that we can't emulate in # GUI or web frontend if os.name == 'posix': for cmd in ('clear', 'more', 'less', 'man'): self.alias_manager.soft_define_alias(cmd, cmd) def __init__(self, *args, **kwargs): super(TerminalInteractiveShell, self).__init__(*args, **kwargs) self.init_prompt_toolkit_cli() self.init_term_title() self.keep_running = True self.debugger_history = InMemoryHistory() def ask_exit(self): self.keep_running = False rl_next_input = None def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED): if display_banner is not DISPLAY_BANNER_DEPRECATED: warn( 'interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2) self.keep_running = True while self.keep_running: print(self.separate_in, end='') try: code = self.prompt_for_code() except EOFError: if (not self.confirm_exit) \ or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'): self.ask_exit() else: if code: self.run_cell(code, store_history=True) def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED): # An extra layer of protection in case someone mashing Ctrl-C breaks # out of our internal code. if display_banner is not DISPLAY_BANNER_DEPRECATED: warn( 'mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2) while True: try: self.interact() break except KeyboardInterrupt as e: print("\n%s escaped interact()\n" % type(e).__name__) finally: # An interrupt during the eventloop will mess up the # internal state of the prompt_toolkit library. # Stopping the eventloop fixes this, see # https://github.com/ipython/ipython/pull/9867 if hasattr(self, '_eventloop'): self._eventloop.stop() self.restore_term_title() _inputhook = None def inputhook(self, context): if self._inputhook is not None: self._inputhook(context) active_eventloop = None def enable_gui(self, gui=None): if gui and (gui != 'inline'): self.active_eventloop, self._inputhook =\ get_inputhook_name_and_func(gui) else: self.active_eventloop = self._inputhook = None # For prompt_toolkit 3.0. We have to create an asyncio event loop with # this inputhook. if PTK3: import asyncio from prompt_toolkit.eventloop import new_eventloop_with_inputhook if gui == 'asyncio': # When we integrate the asyncio event loop, run the UI in the # same event loop as the rest of the code. don't use an actual # input hook. (Asyncio is not made for nesting event loops.) self.pt_loop = asyncio.get_event_loop() elif self._inputhook: # If an inputhook was set, create a new asyncio event loop with # this inputhook for the prompt. self.pt_loop = new_eventloop_with_inputhook(self._inputhook) else: # When there's no inputhook, run the prompt in a separate # asyncio event loop. self.pt_loop = asyncio.new_event_loop() # Run !system commands directly, not through pipes, so terminal programs # work correctly. system = InteractiveShell.system_raw def auto_rewrite_input(self, cmd): """Overridden from the parent class to use fancy rewriting prompt""" if not self.show_rewritten_input: return tokens = self.prompts.rewrite_prompt_tokens() if self.pt_app: print_formatted_text(PygmentsTokens(tokens), end='', style=self.pt_app.app.style) print(cmd) else: prompt = ''.join(s for t, s in tokens) print(prompt, cmd, sep='') _prompts_before = None def switch_doctest_mode(self, mode): """Switch prompts to classic for %doctest_mode""" if mode: self._prompts_before = self.prompts self.prompts = ClassicPrompts(self) elif self._prompts_before: self.prompts = self._prompts_before self._prompts_before = None
class ExecutePreprocessor(Preprocessor): """ Executes all the cells in a notebook """ timeout = Integer(30, allow_none=True, help=dedent(""" The time to wait (in seconds) for output from executions. If a cell execution takes longer, an exception (TimeoutError on python 3+, RuntimeError on python 2) is raised. `None` or `-1` will disable the timeout. If `timeout_func` is set, it overrides `timeout`. """)).tag(config=True) timeout_func = Any(default_value=None, allow_none=True, help=dedent(""" A callable which, when given the cell source as input, returns the time to wait (in seconds) for output from cell executions. If a cell execution takes longer, an exception (TimeoutError on python 3+, RuntimeError on python 2) is raised. Returning `None` or `-1` will disable the timeout for the cell. Not setting `timeout_func` will cause the preprocessor to default to using the `timeout` trait for all cells. The `timeout_func` trait overrides `timeout` if it is not `None`. """)).tag(config=True) interrupt_on_timeout = Bool(False, help=dedent(""" If execution of a cell times out, interrupt the kernel and continue executing other cells rather than throwing an error and stopping. """)).tag(config=True) startup_timeout = Integer(60, help=dedent(""" The time to wait (in seconds) for the kernel to start. If kernel startup takes longer, a RuntimeError is raised. """)).tag(config=True) allow_errors = Bool(False, help=dedent(""" If `False` (default), when a cell raises an error the execution is stopped and a `CellExecutionError` is raised. If `True`, execution errors are ignored and the execution is continued until the end of the notebook. Output from exceptions is included in the cell output in both cases. """)).tag(config=True) force_raise_errors = Bool(False, help=dedent(""" If False (default), errors from executing the notebook can be allowed with a `raises-exception` tag on a single cell, or the `allow_errors` configurable option for all cells. An allowed error will be recorded in notebook output, and execution will continue. If an error occurs when it is not explicitly allowed, a `CellExecutionError` will be raised. If True, `CellExecutionError` will be raised for any error that occurs while executing the notebook. This overrides both the `allow_errors` option and the `raises-exception` cell tag. """)).tag(config=True) extra_arguments = List(Unicode()) kernel_name = Unicode('', help=dedent(""" Name of kernel to use to execute the cells. If not set, use the kernel_spec embedded in the notebook. """)).tag(config=True) raise_on_iopub_timeout = Bool(False, help=dedent(""" If `False` (default), then the kernel will continue waiting for iopub messages until it receives a kernel idle message, or until a timeout occurs, at which point the currently executing cell will be skipped. If `True`, then an error will be raised after the first timeout. This option generally does not need to be used, but may be useful in contexts where there is the possibility of executing notebooks with memory-consuming infinite loops. """)).tag(config=True) store_widget_state = Bool(True, help=dedent(""" If `True` (default), then the state of the Jupyter widgets created at the kernel will be stored in the metadata of the notebook. """)).tag(config=True) iopub_timeout = Integer(4, allow_none=False, help=dedent(""" The time to wait (in seconds) for IOPub output. This generally doesn't need to be set, but on some slow networks (such as CI systems) the default timeout might not be long enough to get all messages. """)).tag(config=True) shutdown_kernel = Enum(['graceful', 'immediate'], default_value='graceful', help=dedent(""" If `graceful` (default), then the kernel is given time to clean up after executing all cells, e.g., to execute its `atexit` hooks. If `immediate`, then the kernel is signaled to immediately terminate. """)).tag(config=True) ipython_hist_file = Unicode( default_value=':memory:', help= """Path to file to use for SQLite history database for an IPython kernel. The specific value `:memory:` (including the colon at both end but not the back ticks), avoids creating a history file. Otherwise, IPython will create a history file for each kernel. When running kernels simultaneously (e.g. via multiprocessing) saving history a single SQLite file can result in database errors, so using `:memory:` is recommended in non-interactive contexts. """).tag(config=True) kernel_manager_class = Type(config=True, help='The kernel manager class to use.') @default('kernel_manager_class') def _kernel_manager_class_default(self): """Use a dynamic default to avoid importing jupyter_client at startup""" try: from jupyter_client import KernelManager except ImportError: raise ImportError( "`nbconvert --execute` requires the jupyter_client package: `pip install jupyter_client`" ) return KernelManager _display_id_map = Dict(help=dedent(""" mapping of locations of outputs with a given display_id tracks cell index and output index within cell.outputs for each appearance of the display_id { 'display_id': { cell_idx: [output_idx,] } } """)) def start_new_kernel(self, **kwargs): """Creates a new kernel manager and kernel client. Parameters ---------- kwargs : Any options for `self.kernel_manager_class.start_kernel()`. Because that defaults to KernelManager, this will likely include options accepted by `KernelManager.start_kernel()``, which includes `cwd`. Returns ------- km : KernelManager A kernel manager as created by self.kernel_manager_class. kc : KernelClient Kernel client as created by the kernel manager `km`. """ if not self.kernel_name: self.kernel_name = self.nb.metadata.get('kernelspec', {}).get('name', 'python') km = self.kernel_manager_class(kernel_name=self.kernel_name, config=self.config) if km.ipykernel and self.ipython_hist_file: self.extra_arguments += [ '--HistoryManager.hist_file={}'.format(self.ipython_hist_file) ] km.start_kernel(extra_arguments=self.extra_arguments, **kwargs) kc = km.client() kc.start_channels() try: kc.wait_for_ready(timeout=self.startup_timeout) except RuntimeError: kc.stop_channels() km.shutdown_kernel() raise kc.allow_stdin = False return km, kc @contextmanager def setup_preprocessor(self, nb, resources, km=None, **kwargs): """ Context manager for setting up the class to execute a notebook. The assigns `nb` to `self.nb` where it will be modified in-place. It also creates and assigns the Kernel Manager (`self.km`) and Kernel Client(`self.kc`). It is intended to yield to a block that will execute codeself. When control returns from the yield it stops the client's zmq channels, shuts down the kernel, and removes the now unused attributes. Parameters ---------- nb : NotebookNode Notebook being executed. resources : dictionary Additional resources used in the conversion process. For example, passing ``{'metadata': {'path': run_path}}`` sets the execution path to ``run_path``. km : KernerlManager (optional) Optional kernel manager. If none is provided, a kernel manager will be created. Returns ------- nb : NotebookNode The executed notebook. resources : dictionary Additional resources used in the conversion process. """ path = resources.get('metadata', {}).get('path', '') or None self.nb = nb # clear display_id map self._display_id_map = {} self.widget_state = {} self.widget_buffers = {} if km is None: kwargs["cwd"] = path self.km, self.kc = self.start_new_kernel(**kwargs) try: # Yielding unbound args for more easier understanding and downstream consumption yield nb, self.km, self.kc finally: self.kc.stop_channels() self.km.shutdown_kernel( now=self.shutdown_kernel == 'immediate') for attr in ['nb', 'km', 'kc']: delattr(self, attr) else: self.km = km if not km.has_kernel: km.start_kernel(extra_arguments=self.extra_arguments, **kwargs) self.kc = km.client() self.kc.start_channels() try: self.kc.wait_for_ready(timeout=self.startup_timeout) except RuntimeError: self.kc.stop_channels() raise self.kc.allow_stdin = False try: yield nb, self.km, self.kc finally: for attr in ['nb', 'km', 'kc']: delattr(self, attr) def preprocess(self, nb, resources, km=None): """ Preprocess notebook executing each code cell. The input argument `nb` is modified in-place. Parameters ---------- nb : NotebookNode Notebook being executed. resources : dictionary Additional resources used in the conversion process. For example, passing ``{'metadata': {'path': run_path}}`` sets the execution path to ``run_path``. km: KernelManager (optional) Optional kernel manager. If none is provided, a kernel manager will be created. Returns ------- nb : NotebookNode The executed notebook. resources : dictionary Additional resources used in the conversion process. """ with self.setup_preprocessor(nb, resources, km=km): self.log.info("Executing notebook with kernel: %s" % self.kernel_name) nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources) info_msg = self._wait_for_reply(self.kc.kernel_info()) nb.metadata['language_info'] = info_msg['content']['language_info'] self.set_widgets_metadata() return nb, resources def set_widgets_metadata(self): if self.widget_state: self.nb.metadata.widgets = { 'application/vnd.jupyter.widget-state+json': { 'state': { model_id: _serialize_widget_state(state) for model_id, state in self.widget_state.items() if '_model_name' in state }, 'version_major': 2, 'version_minor': 0, } } for key, widget in self.nb.metadata.widgets[ 'application/vnd.jupyter.widget-state+json'][ 'state'].items(): buffers = self.widget_buffers.get(key) if buffers: widget['buffers'] = buffers def preprocess_cell(self, cell, resources, cell_index): """ Executes a single code cell. See base.py for details. To execute all cells see :meth:`preprocess`. """ if cell.cell_type != 'code' or not cell.source.strip(): return cell, resources reply, outputs = self.run_cell(cell, cell_index) # Backwards compatibility for processes that wrap run_cell cell.outputs = outputs cell_allows_errors = (self.allow_errors or "raises-exception" in cell.metadata.get( "tags", [])) if self.force_raise_errors or not cell_allows_errors: for out in cell.outputs: if out.output_type == 'error': raise CellExecutionError.from_cell_and_msg(cell, out) if (reply is not None) and reply['content']['status'] == 'error': raise CellExecutionError.from_cell_and_msg( cell, reply['content']) return cell, resources def _update_display_id(self, display_id, msg): """Update outputs with a given display_id""" if display_id not in self._display_id_map: self.log.debug("display id %r not in %s", display_id, self._display_id_map) return if msg['header']['msg_type'] == 'update_display_data': msg['header']['msg_type'] = 'display_data' try: out = output_from_msg(msg) except ValueError: self.log.error("unhandled iopub msg: " + msg['msg_type']) return for cell_idx, output_indices in self._display_id_map[display_id].items( ): cell = self.nb['cells'][cell_idx] outputs = cell['outputs'] for output_idx in output_indices: outputs[output_idx]['data'] = out['data'] outputs[output_idx]['metadata'] = out['metadata'] def _poll_for_reply(self, msg_id, cell=None, timeout=None): try: # check with timeout if kernel is still alive msg = self.kc.shell_channel.get_msg(timeout=timeout) if msg['parent_header'].get('msg_id') == msg_id: return msg except Empty: # received no message, check if kernel is still alive self._check_alive() # kernel still alive, wait for a message def _get_timeout(self, cell): if self.timeout_func is not None and cell is not None: timeout = self.timeout_func(cell) else: timeout = self.timeout if not timeout or timeout < 0: timeout = None return timeout def _handle_timeout(self): self.log.error("Timeout waiting for execute reply (%is)." % self.timeout) if self.interrupt_on_timeout: self.log.error("Interrupting kernel") self.km.interrupt_kernel() else: raise TimeoutError("Cell execution timed out") def _check_alive(self): if not self.kc.is_alive(): self.log.error("Kernel died while waiting for execute reply.") raise DeadKernelError("Kernel died") def _wait_for_reply(self, msg_id, cell=None): # wait for finish, with timeout timeout = self._get_timeout(cell) cummulative_time = 0 timeout_interval = 5 while True: try: msg = self.kc.shell_channel.get_msg(timeout=timeout_interval) except Empty: self._check_alive() cummulative_time += timeout_interval if timeout and cummulative_time > timeout: self._handle_timeout() break else: if msg['parent_header'].get('msg_id') == msg_id: return msg def _timeout_with_deadline(self, timeout, deadline): if deadline is not None and deadline - monotonic() < timeout: timeout = deadline - monotonic() if timeout < 0: timeout = 0 return timeout def _passed_deadline(self, deadline): if deadline is not None and deadline - monotonic() <= 0: self._handle_timeout() return True return False def run_cell(self, cell, cell_index=0): parent_msg_id = self.kc.execute(cell.source) self.log.debug("Executing cell:\n%s", cell.source) exec_timeout = self._get_timeout(cell) deadline = None if exec_timeout is not None: deadline = monotonic() + exec_timeout cell.outputs = [] self.clear_before_next_output = False # This loop resolves #659. By polling iopub_channel's and shell_channel's # output we avoid dropping output and important signals (like idle) from # iopub_channel. Prior to this change, iopub_channel wasn't polled until # after exec_reply was obtained from shell_channel, leading to the # aforementioned dropped data. # These two variables are used to track what still needs polling: # more_output=true => continue to poll the iopub_channel more_output = True # polling_exec_reply=true => continue to poll the shell_channel polling_exec_reply = True while more_output or polling_exec_reply: if polling_exec_reply: if self._passed_deadline(deadline): polling_exec_reply = False continue # Avoid exceeding the execution timeout (deadline), but stop # after at most 1s so we can poll output from iopub_channel. timeout = self._timeout_with_deadline(1, deadline) exec_reply = self._poll_for_reply(parent_msg_id, cell, timeout) if exec_reply is not None: polling_exec_reply = False if more_output: try: timeout = self.iopub_timeout if polling_exec_reply: # Avoid exceeding the execution timeout (deadline) while # polling for output. timeout = self._timeout_with_deadline( timeout, deadline) msg = self.kc.iopub_channel.get_msg(timeout=timeout) except Empty: if polling_exec_reply: # Still waiting for execution to finish so we expect that # output may not always be produced yet. continue if self.raise_on_iopub_timeout: raise TimeoutError("Timeout waiting for IOPub output") else: self.log.warning("Timeout waiting for IOPub output") more_output = False continue if msg['parent_header'].get('msg_id') != parent_msg_id: # not an output from our execution continue try: # Will raise CellExecutionComplete when completed self.process_message(msg, cell, cell_index) except CellExecutionComplete: more_output = False # Return cell.outputs still for backwards compatibility return exec_reply, cell.outputs def process_message(self, msg, cell, cell_index): """ Processes a kernel message, updates cell state, and returns the resulting output object that was appended to cell.outputs. The input argument `cell` is modified in-place. Parameters ---------- msg : dict The kernel message being processed. cell : nbformat.NotebookNode The cell which is currently being processed. cell_index : int The position of the cell within the notebook object. Returns ------- output : dict The execution output payload (or None for no output). Raises ------ CellExecutionComplete Once a message arrives which indicates computation completeness. """ msg_type = msg['msg_type'] self.log.debug("msg_type: %s", msg_type) content = msg['content'] self.log.debug("content: %s", content) display_id = content.get('transient', {}).get('display_id', None) if display_id and msg_type in { 'execute_result', 'display_data', 'update_display_data' }: self._update_display_id(display_id, msg) # set the prompt number for the input and the output if 'execution_count' in content: cell['execution_count'] = content['execution_count'] if msg_type == 'status': if content['execution_state'] == 'idle': raise CellExecutionComplete() elif msg_type == 'clear_output': self.clear_output(cell.outputs, msg, cell_index) elif msg_type.startswith('comm'): self.handle_comm_msg(cell.outputs, msg, cell_index) # Check for remaining messages we don't process elif msg_type not in ['execute_input', 'update_display_data']: # Assign output as our processed "result" return self.output(cell.outputs, msg, display_id, cell_index) def output(self, outs, msg, display_id, cell_index): msg_type = msg['msg_type'] try: out = output_from_msg(msg) except ValueError: self.log.error("unhandled iopub msg: " + msg_type) return if self.clear_before_next_output: self.log.debug('Executing delayed clear_output') outs[:] = [] self.clear_display_id_mapping(cell_index) self.clear_before_next_output = False if display_id: # record output index in: # _display_id_map[display_id][cell_idx] cell_map = self._display_id_map.setdefault(display_id, {}) output_idx_list = cell_map.setdefault(cell_index, []) output_idx_list.append(len(outs)) outs.append(out) return out def clear_output(self, outs, msg, cell_index): content = msg['content'] if content.get('wait'): self.log.debug('Wait to clear output') self.clear_before_next_output = True else: self.log.debug('Immediate clear output') outs[:] = [] self.clear_display_id_mapping(cell_index) def clear_display_id_mapping(self, cell_index): for display_id, cell_map in self._display_id_map.items(): if cell_index in cell_map: cell_map[cell_index] = [] def handle_comm_msg(self, outs, msg, cell_index): content = msg['content'] data = content['data'] if self.store_widget_state and 'state' in data: # ignore custom msg'es self.widget_state.setdefault(content['comm_id'], {}).update(data['state']) if 'buffer_paths' in data and data['buffer_paths']: self.widget_buffers[content['comm_id']] = _get_buffer_data(msg)
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin): """ A Qt frontend for a generic Python kernel. """ # The text to show when the kernel is (re)started. banner = Unicode(config=True) kernel_banner = Unicode() # Whether to show the banner _display_banner = Bool(False) # An option and corresponding signal for overriding the default kernel # interrupt behavior. custom_interrupt = Bool(False) custom_interrupt_requested = QtCore.Signal() # An option and corresponding signals for overriding the default kernel # restart behavior. custom_restart = Bool(False) custom_restart_kernel_died = QtCore.Signal(float) custom_restart_requested = QtCore.Signal() # Whether to automatically show calltips on open-parentheses. enable_calltips = Bool( True, config=True, help="Whether to draw information calltips on open-parentheses.") clear_on_kernel_restart = Bool( True, config=True, help="Whether to clear the console when the kernel is restarted") confirm_restart = Bool( True, config=True, help="Whether to ask for user confirmation when restarting kernel") lexer_class = DottedObjectName(config=True, help="The pygments lexer class to use.") is_complete_timeout = Float( 0.25, config=True, help="Seconds to wait for is_complete replies from the kernel.") def _lexer_class_changed(self, name, old, new): lexer_class = import_item(new) self.lexer = lexer_class() def _lexer_class_default(self): if py3compat.PY3: return 'pygments.lexers.Python3Lexer' else: return 'pygments.lexers.PythonLexer' lexer = Any() def _lexer_default(self): lexer_class = import_item(self.lexer_class) return lexer_class() # Emitted when a user visible 'execute_request' has been submitted to the # kernel from the FrontendWidget. Contains the code to be executed. executing = QtCore.Signal(object) # Emitted when a user-visible 'execute_reply' has been received from the # kernel and processed by the FrontendWidget. Contains the response message. executed = QtCore.Signal(object) # Emitted when an exit request has been received from the kernel. exit_requested = QtCore.Signal(object) _CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos']) _CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos']) _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) _local_kernel = False _highlighter = Instance(FrontendHighlighter, allow_none=True) #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(FrontendWidget, self).__init__(*args, **kw) # FIXME: remove this when PySide min version is updated past 1.0.7 # forcefully disable calltips if PySide is < 1.0.7, because they crash if qt.QT_API == qt.QT_API_PYSIDE: import PySide if PySide.__version_info__ < (1, 0, 7): self.log.warn( "PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__) self.enable_calltips = False # FrontendWidget protected variables. self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None) self._hidden = False self._highlighter = FrontendHighlighter(self, lexer=self.lexer) self._kernel_manager = None self._kernel_client = None self._request_info = {} self._request_info['execute'] = {} self._callback_dict = {} self._display_banner = True # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Configure actions. action = self._copy_raw_action key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C action.setEnabled(False) action.setShortcut(QtGui.QKeySequence(key)) action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut) action.triggered.connect(self.copy_raw) self.copy_available.connect(action.setEnabled) self.addAction(action) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Set flag for whether we are connected via localhost. self._local_kernel = kw.get('local_kernel', FrontendWidget._local_kernel) # Whether or not a clear_output call is pending new output. self._pending_clearoutput = False #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- def copy(self): """ Copy the currently selected text to the clipboard, removing prompts. """ if self._page_control is not None and self._page_control.hasFocus(): self._page_control.copy() elif self._control.hasFocus(): text = self._control.textCursor().selection().toPlainText() if text: # Remove prompts. lines = text.splitlines() lines = map(transform_classic_prompt, lines) lines = map(transform_ipy_prompt, lines) text = '\n'.join(lines) was_newline = text[-1] == '\n' if was_newline: # user doesn't need newline text = text[:-1] QtGui.QApplication.clipboard().setText(text) else: self.log.debug("frontend widget : unknown copy target") #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _is_complete(self, source, interactive): """ Returns whether 'source' can be completely processed and a new prompt created. When triggered by an Enter/Return key press, 'interactive' is True; otherwise, it is False. Returns ------- (complete, indent): (bool, str) complete is a bool, indicating whether the input is complete or not. indent is the current indentation string for autoindent. If complete is True, indent will be '', and should be ignored. """ kc = self.blocking_client if kc is None: self.log.warn("No blocking client to make is_complete requests") return False, u'' msg_id = kc.is_complete(source) while True: try: reply = kc.shell_channel.get_msg( block=True, timeout=self.is_complete_timeout) except Empty: # assume incomplete output if we get no reply in time return False, u'' if reply['parent_header'].get('msg_id', None) == msg_id: status = reply['content'].get('status', u'complete') indent = reply['content'].get('indent', u'') return status != 'incomplete', indent def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ msg_id = self.kernel_client.execute(source, hidden) self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'user') self._hidden = hidden if not hidden: self.executing.emit(source) def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. text = self._get_input_buffer_cursor_line() if text is None: return False complete = bool(text[:self._get_input_buffer_cursor_column()].strip()) if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _context_menu_make(self, pos): """ Reimplemented to add an action for raw copy. """ menu = super(FrontendWidget, self)._context_menu_make(pos) for before_action in menu.actions(): if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \ QtGui.QKeySequence.ExactMatch: menu.insertAction(before_action, self._copy_raw_action) break return menu def request_interrupt_kernel(self): if self._executing: self.interrupt_kernel() def request_restart_kernel(self): message = 'Are you sure you want to restart the kernel?' self.restart_kernel(message, now=False) def _event_filter_console_keypress(self, event): """ Reimplemented for execution interruption and smart backspace. """ key = event.key() if self._control_key_down(event.modifiers(), include_command=False): if key == QtCore.Qt.Key_C and self._executing: self.request_interrupt_kernel() return True elif key == QtCore.Qt.Key_Period: self.request_restart_kernel() return True elif not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces if key == QtCore.Qt.Key_Backspace: col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(FrontendWidget, self)._event_filter_console_keypress(event) def _insert_continuation_prompt(self, cursor, indent=''): """ Reimplemented for auto-indentation. """ super(FrontendWidget, self)._insert_continuation_prompt(cursor) if indent: cursor.insertText(indent) #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_clear_output(self, msg): """Handle clear output messages.""" if self.include_output(msg): wait = msg['content'].get('wait', True) if wait: self._pending_clearoutput = True else: self.clear_output() def _silent_exec_callback(self, expr, callback): """Silently execute `expr` in the kernel and call `callback` with reply the `expr` is evaluated silently in the kernel (without) output in the frontend. Call `callback` with the `repr <http://docs.python.org/library/functions.html#repr> `_ as first argument Parameters ---------- expr : string valid string to be executed by the kernel. callback : function function accepting one argument, as a string. The string will be the `repr` of the result of evaluating `expr` The `callback` is called with the `repr()` of the result of `expr` as first argument. To get the object, do `eval()` on the passed value. See Also -------- _handle_exec_callback : private method, deal with calling callback with reply """ # generate uuid, which would be used as an indication of whether or # not the unique request originated from here (can use msg id ?) local_uuid = str(uuid.uuid1()) msg_id = self.kernel_client.execute( '', silent=True, user_expressions={local_uuid: expr}) self._callback_dict[local_uuid] = callback self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'silent_exec_callback') def _handle_exec_callback(self, msg): """Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback`` Parameters ---------- msg : raw message send by the kernel containing an `user_expressions` and having a 'silent_exec_callback' kind. Notes ----- This function will look for a `callback` associated with the corresponding message id. Association has been made by `_silent_exec_callback`. `callback` is then called with the `repr()` of the value of corresponding `user_expressions` as argument. `callback` is then removed from the known list so that any message coming again with the same id won't trigger it. """ user_exp = msg['content'].get('user_expressions') if not user_exp: return for expression in user_exp: if expression in self._callback_dict: self._callback_dict.pop(expression)(user_exp[expression]) def _handle_execute_reply(self, msg): """ Handles replies for code execution. """ self.log.debug("execute: %s", msg.get('content', '')) msg_id = msg['parent_header']['msg_id'] info = self._request_info['execute'].get(msg_id) # unset reading flag, because if execute finished, raw_input can't # still be pending. self._reading = False if info and info.kind == 'user' and not self._hidden: # Make sure that all output from the SUB channel has been processed # before writing a new prompt. self.kernel_client.iopub_channel.flush() # Reset the ANSI style information to prevent bad text in stdout # from messing up our colors. We're not a true terminal so we're # allowed to do this. if self.ansi_codes: self._ansi_processor.reset_sgr() content = msg['content'] status = content['status'] if status == 'ok': self._process_execute_ok(msg) elif status == 'error': self._process_execute_error(msg) elif status == 'aborted': self._process_execute_abort(msg) self._show_interpreter_prompt_for_reply(msg) self.executed.emit(msg) self._request_info['execute'].pop(msg_id) elif info and info.kind == 'silent_exec_callback' and not self._hidden: self._handle_exec_callback(msg) self._request_info['execute'].pop(msg_id) else: super(FrontendWidget, self)._handle_execute_reply(msg) def _handle_input_request(self, msg): """ Handle requests for raw_input. """ self.log.debug("input: %s", msg.get('content', '')) if self._hidden: raise RuntimeError( 'Request for raw input during hidden execution.') # Make sure that all output from the SUB channel has been processed # before entering readline mode. self.kernel_client.iopub_channel.flush() def callback(line): self.kernel_client.input(line) if self._reading: self.log.debug( "Got second input request, assuming first was interrupted.") self._reading = False self._readline(msg['content']['prompt'], callback=callback, password=msg['content']['password']) def _kernel_restarted_message(self, died=True): msg = "Kernel died, restarting" if died else "Kernel restarting" self._append_html("<br>%s<hr><br>" % msg, before_prompt=False) def _handle_kernel_died(self, since_last_heartbeat): """Handle the kernel's death (if we do not own the kernel). """ self.log.warn("kernel died: %s", since_last_heartbeat) if self.custom_restart: self.custom_restart_kernel_died.emit(since_last_heartbeat) else: self._kernel_restarted_message(died=True) self.reset() def _handle_kernel_restarted(self, died=True): """Notice that the autorestarter restarted the kernel. There's nothing to do but show a message. """ self.log.warn("kernel restarted") self._kernel_restarted_message(died=died) self.reset() def _handle_inspect_reply(self, rep): """Handle replies for call tips.""" self.log.debug("oinfo: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('call_tip') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): content = rep['content'] if content.get('status') == 'ok' and content.get('found', False): self._call_tip_widget.show_inspect_data(content) def _handle_execute_result(self, msg): """ Handle display hook output. """ self.log.debug("execute_result: %s", msg.get('content', '')) if self.include_output(msg): self.flush_clearoutput() text = msg['content']['data'] self._append_plain_text(text + '\n', before_prompt=True) def _handle_stream(self, msg): """ Handle stdout, stderr, and stdin. """ self.log.debug("stream: %s", msg.get('content', '')) if self.include_output(msg): self.flush_clearoutput() self.append_stream(msg['content']['text']) def _handle_shutdown_reply(self, msg): """ Handle shutdown signal, only if from other console. """ self.log.debug("shutdown: %s", msg.get('content', '')) restart = msg.get('content', {}).get('restart', False) if not self._hidden and not self.from_here(msg): # got shutdown reply, request came from session other than ours if restart: # someone restarted the kernel, handle it self._handle_kernel_restarted(died=False) else: # kernel was shutdown permanently # this triggers exit_requested if the kernel was local, # and a dialog if the kernel was remote, # so we don't suddenly clear the qtconsole without asking. if self._local_kernel: self.exit_requested.emit(self) else: title = self.window().windowTitle() reply = QtGui.QMessageBox.question( self, title, "Kernel has been shutdown permanently. " "Close the Console?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.Yes: self.exit_requested.emit(self) def _handle_status(self, msg): """Handle status message""" # This is where a busy/idle indicator would be triggered, # when we make one. state = msg['content'].get('execution_state', '') if state == 'starting': # kernel started while we were running if self._executing: self._handle_kernel_restarted(died=True) elif state == 'idle': pass elif state == 'busy': pass def _started_channels(self): """ Called when the KernelManager channels have started listening or when the frontend is assigned an already listening KernelManager. """ self.reset(clear=True) #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def copy_raw(self): """ Copy the currently selected text to the clipboard without attempting to remove prompts or otherwise alter the text. """ self._control.copy() def interrupt_kernel(self): """ Attempts to interrupt the running kernel. Also unsets _reading flag, to avoid runtime errors if raw_input is called again. """ if self.custom_interrupt: self._reading = False self.custom_interrupt_requested.emit() elif self.kernel_manager: self._reading = False self.kernel_manager.interrupt_kernel() else: self._append_plain_text( 'Cannot interrupt a kernel I did not start.\n') def reset(self, clear=False): """ Resets the widget to its initial state if ``clear`` parameter is True, otherwise prints a visual indication of the fact that the kernel restarted, but does not clear the traces from previous usage of the kernel before it was restarted. With ``clear=True``, it is similar to ``%clear``, but also re-writes the banner and aborts execution if necessary. """ if self._executing: self._executing = False self._request_info['execute'] = {} self._reading = False self._highlighter.highlighting_on = False if clear: self._control.clear() if self._display_banner: self._append_plain_text(self.banner) if self.kernel_banner: self._append_plain_text(self.kernel_banner) # update output marker for stdout/stderr, so that startup # messages appear after banner: self._append_before_prompt_pos = self._get_cursor().position() self._show_interpreter_prompt() def restart_kernel(self, message, now=False): """ Attempts to restart the running kernel. """ # FIXME: now should be configurable via a checkbox in the dialog. Right # now at least the heartbeat path sets it to True and the manual restart # to False. But those should just be the pre-selected states of a # checkbox that the user could override if so desired. But I don't know # enough Qt to go implementing the checkbox now. if self.custom_restart: self.custom_restart_requested.emit() return if self.kernel_manager: # Pause the heart beat channel to prevent further warnings. self.kernel_client.hb_channel.pause() # Prompt the user to restart the kernel. Un-pause the heartbeat if # they decline. (If they accept, the heartbeat will be un-paused # automatically when the kernel is restarted.) if self.confirm_restart: buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No result = QtGui.QMessageBox.question(self, 'Restart kernel?', message, buttons) do_restart = result == QtGui.QMessageBox.Yes else: # confirm_restart is False, so we don't need to ask user # anything, just do the restart do_restart = True if do_restart: try: self.kernel_manager.restart_kernel(now=now) except RuntimeError as e: self._append_plain_text('Error restarting kernel: %s\n' % e, before_prompt=True) else: self._append_html( "<br>Restarting kernel...\n<hr><br>", before_prompt=True, ) else: self.kernel_client.hb_channel.unpause() else: self._append_plain_text( 'Cannot restart a Kernel I did not start\n', before_prompt=True) def append_stream(self, text): """Appends text to the output stream.""" # Most consoles treat tabs as being 8 space characters. Convert tabs # to spaces so that output looks as expected regardless of this # widget's tab width. text = text.expandtabs(8) self._append_plain_text(text, before_prompt=True) self._control.moveCursor(QtGui.QTextCursor.End) def flush_clearoutput(self): """If a clearoutput is pending, execute it.""" if self._pending_clearoutput: self._pending_clearoutput = False self.clear_output() def clear_output(self): """Clears the current line of output.""" cursor = self._control.textCursor() cursor.beginEditBlock() cursor.movePosition(cursor.StartOfLine, cursor.KeepAnchor) cursor.insertText('') cursor.endEditBlock() #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _auto_call_tip(self): """Trigger call tip automatically on open parenthesis Call tips can be requested explcitly with `_call_tip`. """ cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) == '(': # trigger auto call tip on open paren self._call_tip() def _call_tip(self): """Shows a call tip, if appropriate, at the current cursor location.""" # Decide if it makes sense to show a call tip if not self.enable_calltips or not self.kernel_client.shell_channel.is_alive( ): return False cursor_pos = self._get_input_buffer_cursor_pos() code = self.input_buffer # Send the metadata request to the kernel msg_id = self.kernel_client.inspect(code, cursor_pos) pos = self._get_cursor().position() self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) return True def _complete(self): """ Performs completion at the current cursor location. """ # Send the completion request to the kernel msg_id = self.kernel_client.complete( code=self.input_buffer, cursor_pos=self._get_input_buffer_cursor_pos(), ) pos = self._get_cursor().position() info = self._CompletionRequest(msg_id, pos) self._request_info['complete'] = info def _process_execute_abort(self, msg): """ Process a reply for an aborted execution request. """ self._append_plain_text("ERROR: execution aborted\n") def _process_execute_error(self, msg): """ Process a reply for an execution request that resulted in an error. """ content = msg['content'] # If a SystemExit is passed along, this means exit() was called - also # all the ipython %exit magic syntax of '-k' to be used to keep # the kernel running if content['ename'] == 'SystemExit': keepkernel = content['evalue'] == '-k' or content[ 'evalue'] == 'True' self._keep_kernel_on_exit = keepkernel self.exit_requested.emit(self) else: traceback = ''.join(content['traceback']) self._append_plain_text(traceback) def _process_execute_ok(self, msg): """ Process a reply for a successful execution request. """ payload = msg['content'].get('payload', []) for item in payload: if not self._process_execute_payload(item): warning = 'Warning: received unknown payload of type %s' print(warning % repr(item['source'])) def _process_execute_payload(self, item): """ Process a single payload item from the list of payload items in an execution reply. Returns whether the payload was handled. """ # The basic FrontendWidget doesn't handle payloads, as they are a # mechanism for going beyond the standard Python interpreter model. return False def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self._show_prompt('>>> ') def _show_interpreter_prompt_for_reply(self, msg): """ Shows a prompt for the interpreter given an 'execute_reply' message. """ self._show_interpreter_prompt() #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._auto_call_tip() #------ Trait default initializers ----------------------------------------- def _banner_default(self): """ Returns the standard Python banner. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform)