class Dock(Box): """Displays multiple widgets in a dock""" _view_name = Unicode('DockView', sync=True) description = Unicode(sync=True)
class IPythonWidget(FrontendWidget): """ A FrontendWidget for an IPython kernel. """ # If set, the 'custom_edit_requested(str, int)' signal will be emitted when # an editor is needed for a file. This overrides 'editor' and 'editor_line' # settings. custom_edit = Bool(False) custom_edit_requested = QtCore.Signal(object, object) editor = Unicode(default_editor, config=True, help=""" A command for invoking a system text editor. If the string contains a {filename} format specifier, it will be used. Otherwise, the filename will be appended to the end the command. """) editor_line = Unicode(config=True, help=""" The editor command to use when a specific line number is requested. The string should contain two format specifiers: {line} and {filename}. If this parameter is not specified, the line number option to the %edit magic will be ignored. """) style_sheet = Unicode(config=True, help=""" A CSS stylesheet. The stylesheet can contain classes for: 1. Qt: QPlainTextEdit, QFrame, QWidget, etc 2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter) 3. IPython: .error, .in-prompt, .out-prompt, etc """) syntax_style = Unicode(config=True, help=""" If not empty, use this Pygments style for syntax highlighting. Otherwise, the style sheet is queried for Pygments style information. """) # Prompts. in_prompt = Unicode(default_in_prompt, config=True) out_prompt = Unicode(default_out_prompt, config=True) input_sep = Unicode(default_input_sep, config=True) output_sep = Unicode(default_output_sep, config=True) output_sep2 = Unicode(default_output_sep2, config=True) # FrontendWidget protected class variables. _input_splitter_class = IPythonInputSplitter _prompt_transformer = IPythonInputSplitter(physical_line_transforms=[ipy_prompt()], logical_line_transforms=[], python_line_transforms=[], ) # IPythonWidget protected class variables. _PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number']) _payload_source_edit = 'edit_magic' _payload_source_exit = 'ask_exit' _payload_source_next_input = 'set_next_input' _payload_source_page = 'page' _retrying_history_request = False _starting = False #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(IPythonWidget, self).__init__(*args, **kw) # IPythonWidget protected variables. self._payload_handlers = { self._payload_source_edit : self._handle_payload_edit, self._payload_source_exit : self._handle_payload_exit, self._payload_source_page : self._handle_payload_page, self._payload_source_next_input : self._handle_payload_next_input } self._previous_prompt_obj = None self._keep_kernel_on_exit = None # Initialize widget styling. if self.style_sheet: self._style_sheet_changed() self._syntax_style_changed() else: self.set_default_style() self._guiref_loaded = False #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_complete_reply(self, rep): """ Reimplemented to support IPython's improved completion machinery. """ self.log.debug("complete: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('complete') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): content = rep['content'] matches = content['matches'] start = content['cursor_start'] end = content['cursor_end'] offset = end - start # Move the cursor to the start of the match and complete. cursor.movePosition(QtGui.QTextCursor.Left, n=offset) self._complete_with_items(cursor, matches) def _handle_execute_reply(self, msg): """ Reimplemented to support prompt requests. """ msg_id = msg['parent_header'].get('msg_id') info = self._request_info['execute'].get(msg_id) if info and info.kind == 'prompt': content = msg['content'] if content['status'] == 'aborted': self._show_interpreter_prompt() else: number = content['execution_count'] + 1 self._show_interpreter_prompt(number) self._request_info['execute'].pop(msg_id) else: super(IPythonWidget, self)._handle_execute_reply(msg) def _handle_history_reply(self, msg): """ Implemented to handle history tail replies, which are only supported by the IPython kernel. """ content = msg['content'] if 'history' not in content: self.log.error("History request failed: %r"%content) if content.get('status', '') == 'aborted' and \ not self._retrying_history_request: # a *different* action caused this request to be aborted, so # we should try again. self.log.error("Retrying aborted history request") # prevent multiple retries of aborted requests: self._retrying_history_request = True # wait out the kernel's queue flush, which is currently timed at 0.1s time.sleep(0.25) self.kernel_client.shell_channel.history(hist_access_type='tail',n=1000) else: self._retrying_history_request = False return # reset retry flag self._retrying_history_request = False history_items = content['history'] self.log.debug("Received history reply with %i entries", len(history_items)) items = [] last_cell = u"" for _, _, cell in history_items: cell = cell.rstrip() if cell != last_cell: items.append(cell) last_cell = cell self._set_history(items) def _handle_execute_result(self, msg): """ Reimplemented for IPython-style "display hook". """ self.log.debug("execute_result: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): self.flush_clearoutput() content = msg['content'] prompt_number = content.get('execution_count', 0) data = content['data'] if 'text/plain' in data: self._append_plain_text(self.output_sep, True) self._append_html(self._make_out_prompt(prompt_number), True) text = data['text/plain'] # If the repr is multiline, make sure we start on a new line, # so that its lines are aligned. if "\n" in text and not self.output_sep.endswith("\n"): self._append_plain_text('\n', True) self._append_plain_text(text + self.output_sep2, True) def _handle_display_data(self, msg): """ The base handler for the ``display_data`` message. """ self.log.debug("display: %s", msg.get('content', '')) # For now, we don't display data from other frontends, but we # eventually will as this allows all frontends to monitor the display # data. But we need to figure out how to handle this in the GUI. if not self._hidden and self._is_from_this_session(msg): self.flush_clearoutput() data = msg['content']['data'] metadata = msg['content']['metadata'] # In the regular IPythonWidget, we simply print the plain text # representation. if 'text/plain' in data: text = data['text/plain'] self._append_plain_text(text, True) # This newline seems to be needed for text and html output. self._append_plain_text(u'\n', True) def _handle_kernel_info_reply(self, rep): """Handle kernel info replies.""" content = rep['content'] if not self._guiref_loaded: if content.get('language') == 'python': self._load_guiref_magic() self._guiref_loaded = True self.kernel_banner = content.get('banner', '') if self._starting: # finish handling started channels self._starting = False super(IPythonWidget, self)._started_channels() def _started_channels(self): """Reimplemented to make a history request and load %guiref.""" self._starting = True # The reply will trigger %guiref load provided language=='python' self.kernel_client.kernel_info() self.kernel_client.shell_channel.history(hist_access_type='tail', n=1000) def _load_guiref_magic(self): """Load %guiref magic.""" self.kernel_client.shell_channel.execute('\n'.join([ "try:", " _usage", "except:", " from IPython.core import usage as _usage", " get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')", " del _usage", ]), silent=True) #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def execute_file(self, path, hidden=False): """ Reimplemented to use the 'run' magic. """ # Use forward slashes on Windows to avoid escaping each separator. if sys.platform == 'win32': path = os.path.normpath(path).replace('\\', '/') # Perhaps we should not be using %run directly, but while we # are, it is necessary to quote or escape filenames containing spaces # or quotes. # In earlier code here, to minimize escaping, we sometimes quoted the # filename with single quotes. But to do this, this code must be # platform-aware, because run uses shlex rather than python string # parsing, so that: # * In Win: single quotes can be used in the filename without quoting, # and we cannot use single quotes to quote the filename. # * In *nix: we can escape double quotes in a double quoted filename, # but can't escape single quotes in a single quoted filename. # So to keep this code non-platform-specific and simple, we now only # use double quotes to quote filenames, and escape when needed: if ' ' in path or "'" in path or '"' in path: path = '"%s"' % path.replace('"', '\\"') self.execute('%%run %s' % path, hidden=hidden) #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _process_execute_error(self, msg): """ Reimplemented for IPython-style traceback formatting. """ content = msg['content'] traceback = '\n'.join(content['traceback']) + '\n' if False: # FIXME: For now, tracebacks come as plain text, so we can't use # the html renderer yet. Once we refactor ultratb to produce # properly styled tracebacks, this branch should be the default traceback = traceback.replace(' ', ' ') traceback = traceback.replace('\n', '<br/>') ename = content['ename'] ename_styled = '<span class="error">%s</span>' % ename traceback = traceback.replace(ename, ename_styled) self._append_html(traceback) else: # This is the fallback for now, using plain text with ansi escapes self._append_plain_text(traceback) def _process_execute_payload(self, item): """ Reimplemented to dispatch payloads to handler methods. """ handler = self._payload_handlers.get(item['source']) if handler is None: # We have no handler for this type of payload, simply ignore it return False else: handler(item) return True def _show_interpreter_prompt(self, number=None): """ Reimplemented for IPython-style prompts. """ # If a number was not specified, make a prompt number request. if number is None: msg_id = self.kernel_client.shell_channel.execute('', silent=True) info = self._ExecutionRequest(msg_id, 'prompt') self._request_info['execute'][msg_id] = info return # Show a new prompt and save information about it so that it can be # updated later if the prompt number turns out to be wrong. self._prompt_sep = self.input_sep self._show_prompt(self._make_in_prompt(number), html=True) block = self._control.document().lastBlock() length = len(self._prompt) self._previous_prompt_obj = self._PromptBlock(block, length, number) # Update continuation prompt to reflect (possibly) new prompt length. self._set_continuation_prompt( self._make_continuation_prompt(self._prompt), html=True) def _show_interpreter_prompt_for_reply(self, msg): """ Reimplemented for IPython-style prompts. """ # Update the old prompt number if necessary. content = msg['content'] # abort replies do not have any keys: if content['status'] == 'aborted': if self._previous_prompt_obj: previous_prompt_number = self._previous_prompt_obj.number else: previous_prompt_number = 0 else: previous_prompt_number = content['execution_count'] if self._previous_prompt_obj and \ self._previous_prompt_obj.number != previous_prompt_number: block = self._previous_prompt_obj.block # Make sure the prompt block has not been erased. if block.isValid() and block.text(): # Remove the old prompt and insert a new prompt. cursor = QtGui.QTextCursor(block) cursor.movePosition(QtGui.QTextCursor.Right, QtGui.QTextCursor.KeepAnchor, self._previous_prompt_obj.length) prompt = self._make_in_prompt(previous_prompt_number) self._prompt = self._insert_html_fetching_plain_text( cursor, prompt) # When the HTML is inserted, Qt blows away the syntax # highlighting for the line, so we need to rehighlight it. self._highlighter.rehighlightBlock(cursor.block()) self._previous_prompt_obj = None # Show a new prompt with the kernel's estimated prompt number. self._show_interpreter_prompt(previous_prompt_number + 1) #--------------------------------------------------------------------------- # 'IPythonWidget' interface #--------------------------------------------------------------------------- def set_default_style(self, colors='lightbg'): """ Sets the widget style to the class defaults. Parameters ---------- colors : str, optional (default lightbg) Whether to use the default IPython light background or dark background or B&W style. """ colors = colors.lower() if colors=='lightbg': self.style_sheet = styles.default_light_style_sheet self.syntax_style = styles.default_light_syntax_style elif colors=='linux': self.style_sheet = styles.default_dark_style_sheet self.syntax_style = styles.default_dark_syntax_style elif colors=='nocolor': self.style_sheet = styles.default_bw_style_sheet self.syntax_style = styles.default_bw_syntax_style else: raise KeyError("No such color scheme: %s"%colors) #--------------------------------------------------------------------------- # 'IPythonWidget' protected interface #--------------------------------------------------------------------------- def _edit(self, filename, line=None): """ Opens a Python script for editing. Parameters ---------- filename : str A path to a local system file. line : int, optional A line of interest in the file. """ if self.custom_edit: self.custom_edit_requested.emit(filename, line) elif not self.editor: self._append_plain_text('No default editor available.\n' 'Specify a GUI text editor in the `IPythonWidget.editor` ' 'configurable to enable the %edit magic') else: try: filename = '"%s"' % filename if line and self.editor_line: command = self.editor_line.format(filename=filename, line=line) else: try: command = self.editor.format() except KeyError: command = self.editor.format(filename=filename) else: command += ' ' + filename except KeyError: self._append_plain_text('Invalid editor command.\n') else: try: Popen(command, shell=True) except OSError: msg = 'Opening editor with command "%s" failed.\n' self._append_plain_text(msg % command) def _make_in_prompt(self, number): """ Given a prompt number, returns an HTML In prompt. """ try: body = self.in_prompt % number except TypeError: # allow in_prompt to leave out number, e.g. '>>> ' body = self.in_prompt return '<span class="in-prompt">%s</span>' % body def _make_continuation_prompt(self, prompt): """ Given a plain text version of an In prompt, returns an HTML continuation prompt. """ end_chars = '...: ' space_count = len(prompt.lstrip('\n')) - len(end_chars) body = ' ' * space_count + end_chars return '<span class="in-prompt">%s</span>' % body def _make_out_prompt(self, number): """ Given a prompt number, returns an HTML Out prompt. """ body = self.out_prompt % number return '<span class="out-prompt">%s</span>' % body #------ Payload handlers -------------------------------------------------- # Payload handlers with a generic interface: each takes the opaque payload # dict, unpacks it and calls the underlying functions with the necessary # arguments. def _handle_payload_edit(self, item): self._edit(item['filename'], item['line_number']) def _handle_payload_exit(self, item): self._keep_kernel_on_exit = item['keepkernel'] self.exit_requested.emit(self) def _handle_payload_next_input(self, item): self.input_buffer = item['text'] def _handle_payload_page(self, item): # Since the plain text widget supports only a very small subset of HTML # and we have no control over the HTML source, we only page HTML # payloads in the rich text widget. data = item['data'] if 'text/html' in data and self.kind == 'rich': self._page(data['text/html'], html=True) else: self._page(data['text/plain'], html=False) #------ Trait change handlers -------------------------------------------- def _style_sheet_changed(self): """ Set the style sheets of the underlying widgets. """ self.setStyleSheet(self.style_sheet) if self._control is not None: self._control.document().setDefaultStyleSheet(self.style_sheet) bg_color = self._control.palette().window().color() self._ansi_processor.set_background_color(bg_color) if self._page_control is not None: self._page_control.document().setDefaultStyleSheet(self.style_sheet) def _syntax_style_changed(self): """ Set the style for the syntax highlighter. """ if self._highlighter is None: # ignore premature calls return if self.syntax_style: self._highlighter.set_style(self.syntax_style) else: self._highlighter.set_style_sheet(self.style_sheet) #------ Trait default initializers ----------------------------------------- def _banner_default(self): return "IPython QtConsole {version}\n".format(version=version)
class HistoryManager(HistoryAccessor): """A class to organize all history-related functionality in one place. """ # Public interface # An instance of the IPython shell we are attached to shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') # Lists to hold processed and raw history. These start with a blank entry # so that we can index them starting from 1 input_hist_parsed = List([""]) input_hist_raw = List([""]) # A list of directories visited during session dir_hist = List() def _dir_hist_default(self): try: return [py3compat.getcwd()] except OSError: return [] # A dict of output history, keyed with ints from the shell's # execution count. output_hist = Dict() # The text/plain repr of outputs. output_hist_reprs = Dict() # The number of the current session in the history database session_number = Integer() db_log_output = Bool( False, config=True, help="Should the history database include output? (default: no)") db_cache_size = Integer( 0, config=True, help= "Write to database every x commands (higher values save disk access & power).\n" "Values of 1 or less effectively disable caching.") # The input and output caches db_input_cache = List() db_output_cache = List() # History saving in separate thread save_thread = Instance('IPython.core.history.HistorySavingThread') try: # Event is a function returning an instance of _Event... save_flag = Instance(threading._Event) except AttributeError: # ...until Python 3.3, when it's a class. save_flag = Instance(threading.Event) # Private interface # Variables used to store the three last inputs from the user. On each new # history update, we populate the user's namespace with these, shifted as # necessary. _i00 = Unicode(u'') _i = Unicode(u'') _ii = Unicode(u'') _iii = Unicode(u'') # A regex matching all forms of the exit command, so that we don't store # them in the history (it's annoying to rewind the first entry and land on # an exit call). _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") def __init__(self, shell=None, config=None, **traits): """Create a new history manager associated with a shell instance. """ # We need a pointer back to the shell for various tasks. super(HistoryManager, self).__init__(shell=shell, config=config, **traits) self.save_flag = threading.Event() self.db_input_cache_lock = threading.Lock() self.db_output_cache_lock = threading.Lock() if self.enabled and self.hist_file != ':memory:': self.save_thread = HistorySavingThread(self) self.save_thread.start() self.new_session() def _get_hist_file_name(self, profile=None): """Get default history file name based on the Shell's profile. The profile parameter is ignored, but must exist for compatibility with the parent class.""" profile_dir = self.shell.profile_dir.location return os.path.join(profile_dir, 'history.sqlite') @needs_sqlite def new_session(self, conn=None): """Get a new session number.""" if conn is None: conn = self.db with conn: cur = conn.execute( """INSERT INTO sessions VALUES (NULL, ?, NULL, NULL, "") """, (datetime.datetime.now(), )) self.session_number = cur.lastrowid def end_session(self): """Close the database session, filling in the end time and line count.""" self.writeout_cache() with self.db: self.db.execute( """UPDATE sessions SET end=?, num_cmds=? WHERE session==?""", (datetime.datetime.now(), len(self.input_hist_parsed) - 1, self.session_number)) self.session_number = 0 def name_session(self, name): """Give the current session a name in the history database.""" with self.db: self.db.execute("UPDATE sessions SET remark=? WHERE session==?", (name, self.session_number)) def reset(self, new_session=True): """Clear the session history, releasing all object references, and optionally open a new session.""" self.output_hist.clear() # The directory history can't be completely empty self.dir_hist[:] = [py3compat.getcwd()] if new_session: if self.session_number: self.end_session() self.input_hist_parsed[:] = [""] self.input_hist_raw[:] = [""] self.new_session() # ------------------------------ # Methods for retrieving history # ------------------------------ def get_session_info(self, session=0): """Get info about a session. Parameters ---------- session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is the previous session. Returns ------- session_id : int Session ID number start : datetime Timestamp for the start of the session. end : datetime Timestamp for the end of the session, or None if IPython crashed. num_cmds : int Number of commands run, or None if IPython crashed. remark : unicode A manually set description. """ if session <= 0: session += self.session_number return super(HistoryManager, self).get_session_info(session=session) def _get_range_session(self, start=1, stop=None, raw=True, output=False): """Get input and output history from the current session. Called by get_range, and takes similar parameters.""" input_hist = self.input_hist_raw if raw else self.input_hist_parsed n = len(input_hist) if start < 0: start += n if not stop or (stop > n): stop = n elif stop < 0: stop += n for i in range(start, stop): if output: line = (input_hist[i], self.output_hist_reprs.get(i)) else: line = input_hist[i] yield (0, i, line) def get_range(self, session=0, start=1, stop=None, raw=True, output=False): """Retrieve input by session. Parameters ---------- session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is previous session. start : int First line to retrieve. stop : int End of line range (excluded from output itself). If None, retrieve to the end of the session. raw : bool If True, return untranslated input output : bool If True, attempt to include output. This will be 'real' Python objects for the current session, or text reprs from previous sessions if db_log_output was enabled at the time. Where no output is found, None is used. Returns ------- entries An iterator over the desired lines. Each line is a 3-tuple, either (session, line, input) if output is False, or (session, line, (input, output)) if output is True. """ if session <= 0: session += self.session_number if session == self.session_number: # Current session return self._get_range_session(start, stop, raw, output) return super(HistoryManager, self).get_range(session, start, stop, raw, output) ## ---------------------------- ## Methods for storing history: ## ---------------------------- def store_inputs(self, line_num, source, source_raw=None): """Store source and raw input in history and create input cache variables ``_i*``. Parameters ---------- line_num : int The prompt number of this input. source : str Python input. source_raw : str, optional If given, this is the raw input without any IPython transformations applied to it. If not given, ``source`` is used. """ if source_raw is None: source_raw = source source = source.rstrip('\n') source_raw = source_raw.rstrip('\n') # do not store exit/quit commands if self._exit_re.match(source_raw.strip()): return self.input_hist_parsed.append(source) self.input_hist_raw.append(source_raw) with self.db_input_cache_lock: self.db_input_cache.append((line_num, source, source_raw)) # Trigger to flush cache and write to DB. if len(self.db_input_cache) >= self.db_cache_size: self.save_flag.set() # update the auto _i variables self._iii = self._ii self._ii = self._i self._i = self._i00 self._i00 = source_raw # hackish access to user namespace to create _i1,_i2... dynamically new_i = '_i%s' % line_num to_main = { '_i': self._i, '_ii': self._ii, '_iii': self._iii, new_i: self._i00 } if self.shell is not None: self.shell.push(to_main, interactive=False) def store_output(self, line_num): """If database output logging is enabled, this saves all the outputs from the indicated prompt number to the database. It's called by run_cell after code has been executed. Parameters ---------- line_num : int The line number from which to save outputs """ if (not self.db_log_output) or (line_num not in self.output_hist_reprs): return output = self.output_hist_reprs[line_num] with self.db_output_cache_lock: self.db_output_cache.append((line_num, output)) if self.db_cache_size <= 1: self.save_flag.set() def _writeout_input_cache(self, conn): with conn: for line in self.db_input_cache: conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)", (self.session_number, ) + line) def _writeout_output_cache(self, conn): with conn: for line in self.db_output_cache: conn.execute("INSERT INTO output_history VALUES (?, ?, ?)", (self.session_number, ) + line) @needs_sqlite def writeout_cache(self, conn=None): """Write any entries in the cache to the database.""" if conn is None: conn = self.db with self.db_input_cache_lock: try: self._writeout_input_cache(conn) except sqlite3.IntegrityError: self.new_session(conn) print("ERROR! Session/line number was not unique in", "database. History logging moved to new session", self.session_number) try: # Try writing to the new session. If this fails, don't # recurse self._writeout_input_cache(conn) except sqlite3.IntegrityError: pass finally: self.db_input_cache = [] with self.db_output_cache_lock: try: self._writeout_output_cache(conn) except sqlite3.IntegrityError: print("!! Session/line number for output was not unique", "in database. Output will not be stored.") finally: self.db_output_cache = []
class AutoHandler(PrefilterHandler): handler_name = Unicode('auto') esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2]) def handle(self, line_info): """Handle lines which can be auto-executed, quoting if requested.""" line = line_info.line ifun = line_info.ifun the_rest = line_info.the_rest pre = line_info.pre esc = line_info.esc continue_prompt = line_info.continue_prompt obj = line_info.ofind(self.shell)['obj'] #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest) # dbg # This should only be active for single-line input! if continue_prompt: return line force_auto = isinstance(obj, IPyAutocall) # User objects sometimes raise exceptions on attribute access other # than AttributeError (we've seen it in the past), so it's safest to be # ultra-conservative here and catch all. try: auto_rewrite = obj.rewrite except Exception: auto_rewrite = True if esc == ESC_QUOTE: # Auto-quote splitting on whitespace newcmd = '%s("%s")' % (ifun, '", "'.join(the_rest.split())) elif esc == ESC_QUOTE2: # Auto-quote whole string newcmd = '%s("%s")' % (ifun, the_rest) elif esc == ESC_PAREN: newcmd = '%s(%s)' % (ifun, ",".join(the_rest.split())) else: # Auto-paren. if force_auto: # Don't rewrite if it is already a call. do_rewrite = not the_rest.startswith('(') else: if not the_rest: # We only apply it to argument-less calls if the autocall # parameter is set to 2. do_rewrite = (self.shell.autocall >= 2) elif the_rest.startswith('[') and hasattr(obj, '__getitem__'): # Don't autocall in this case: item access for an object # which is BOTH callable and implements __getitem__. do_rewrite = False else: do_rewrite = True # Figure out the rewritten command if do_rewrite: if the_rest.endswith(';'): newcmd = '%s(%s);' % (ifun.rstrip(), the_rest[:-1]) else: newcmd = '%s(%s)' % (ifun.rstrip(), the_rest) else: normal_handler = self.prefilter_manager.get_handler_by_name( 'normal') return normal_handler.handle(line_info) # Display the rewritten call if auto_rewrite: self.shell.auto_rewrite_input(newcmd) return newcmd
class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp): name = 'ipython-qtconsole' description = """ The IPython QtConsole. This launches a Console-style application using Qt. It is not a full console, in that launched terminal subprocesses will not be able to accept input. The QtConsole supports various extra features beyond the Terminal IPython shell, such as inline plotting with matplotlib, via: ipython qtconsole --pylab=inline as well as saving your session as HTML, and printing the output. """ examples = _examples classes = [IPythonWidget] + IPythonConsoleApp.classes flags = Dict(flags) aliases = Dict(aliases) frontend_flags = Any(qt_flags) frontend_aliases = Any(qt_aliases) kernel_client_class = QtKernelClient kernel_manager_class = QtKernelManager stylesheet = Unicode('', config=True, help="path to a custom CSS stylesheet") plain = CBool( False, config=True, help= "Use a plaintext widget instead of rich text (plain can't print/save)." ) def _plain_changed(self, name, old, new): kind = 'plain' if new else 'rich' self.config.ConsoleWidget.kind = kind if new: self.widget_factory = IPythonWidget else: self.widget_factory = RichIPythonWidget # the factory for creating a widget widget_factory = Any(RichIPythonWidget) def parse_command_line(self, argv=None): super(IPythonQtConsoleApp, self).parse_command_line(argv) self.build_kernel_argv(argv) def new_frontend_master(self): """ Create and return new frontend attached to new kernel, launched on localhost. """ kernel_manager = self.kernel_manager_class( connection_file=self._new_connection_file(), config=self.config, autorestart=True, ) # start the kernel kwargs = dict() kwargs['extra_arguments'] = self.kernel_argv kernel_manager.start_kernel(**kwargs) kernel_manager.client_factory = self.kernel_client_class kernel_client = kernel_manager.client() kernel_client.start_channels(shell=True, iopub=True) widget = self.widget_factory(config=self.config, local_kernel=True) self.init_colors(widget) widget.kernel_manager = kernel_manager widget.kernel_client = kernel_client widget._existing = False widget._may_close = True widget._confirm_exit = self.confirm_exit return widget def new_frontend_slave(self, current_widget): """Create and return a new frontend attached to an existing kernel. Parameters ---------- current_widget : IPythonWidget The IPythonWidget whose kernel this frontend is to share """ kernel_client = self.kernel_client_class( connection_file=current_widget.kernel_client.connection_file, config=self.config, ) kernel_client.load_connection_file() kernel_client.start_channels() widget = self.widget_factory(config=self.config, local_kernel=False) self.init_colors(widget) widget._existing = True widget._may_close = False widget._confirm_exit = False widget.kernel_client = kernel_client widget.kernel_manager = current_widget.kernel_manager return widget def init_qt_app(self): # separate from qt_elements, because it must run first self.app = QtGui.QApplication([]) def init_qt_elements(self): # Create the widget. base_path = os.path.abspath(os.path.dirname(__file__)) icon_path = os.path.join(base_path, 'resources', 'icon', 'IPythonConsole.svg') self.app.icon = QtGui.QIcon(icon_path) QtGui.QApplication.setWindowIcon(self.app.icon) try: ip = self.config.KernelManager.ip except AttributeError: ip = LOCALHOST local_kernel = (not self.existing) or ip in LOCAL_IPS self.widget = self.widget_factory(config=self.config, local_kernel=local_kernel) self.init_colors(self.widget) self.widget._existing = self.existing self.widget._may_close = not self.existing self.widget._confirm_exit = self.confirm_exit self.widget.kernel_manager = self.kernel_manager self.widget.kernel_client = self.kernel_client self.window = MainWindow( self.app, confirm_exit=self.confirm_exit, new_frontend_factory=self.new_frontend_master, slave_frontend_factory=self.new_frontend_slave, ) self.window.log = self.log self.window.add_tab_with_frontend(self.widget) self.window.init_menu_bar() self.window.setWindowTitle('IPython') def init_colors(self, widget): """Configure the coloring of the widget""" # Note: This will be dramatically simplified when colors # are removed from the backend. # parse the colors arg down to current known labels try: colors = self.config.ZMQInteractiveShell.colors except AttributeError: colors = None try: style = self.config.IPythonWidget.syntax_style except AttributeError: style = None try: sheet = self.config.IPythonWidget.style_sheet except AttributeError: sheet = None # find the value for colors: if colors: colors = colors.lower() if colors in ('lightbg', 'light'): colors = 'lightbg' elif colors in ('dark', 'linux'): colors = 'linux' else: colors = 'nocolor' elif style: if style == 'bw': colors = 'nocolor' elif styles.dark_style(style): colors = 'linux' else: colors = 'lightbg' else: colors = None # Configure the style if style: widget.style_sheet = styles.sheet_from_template(style, colors) widget.syntax_style = style widget._syntax_style_changed() widget._style_sheet_changed() elif colors: # use a default dark/light/bw style widget.set_default_style(colors=colors) if self.stylesheet: # we got an explicit stylesheet if os.path.isfile(self.stylesheet): with open(self.stylesheet) as f: sheet = f.read() else: raise IOError("Stylesheet %r not found." % self.stylesheet) if sheet: widget.style_sheet = sheet widget._style_sheet_changed() def init_signal(self): """allow clean shutdown on sigint""" signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-2)) # need a timer, so that QApplication doesn't block until a real # Qt event fires (can require mouse movement) # timer trick from http://stackoverflow.com/q/4938723/938949 timer = QtCore.QTimer() # Let the interpreter run each 200 ms: timer.timeout.connect(lambda: None) timer.start(200) # hold onto ref, so the timer doesn't get cleaned up self._sigint_timer = timer @catch_config_error def initialize(self, argv=None): self.init_qt_app() super(IPythonQtConsoleApp, self).initialize(argv) IPythonConsoleApp.initialize(self, argv) self.init_qt_elements() self.init_signal() def start(self): # draw the window self.window.show() self.window.raise_() # Start the application main loop. self.app.exec_()
class MetaManager(NBXContentsManager): """ Holds NotebookManager classes and routes calls to the appropiate manager. """ debug = Bool(True) file_dirs = Dict(config=True, help="Dict of alias, path") bundle_dirs = Dict(config=True, help="BundleNBManager. Dict of alias, path") github_accounts = List( Tuple, config=True, help="List of Tuple(github_account, github_password)") manager_middleware = Dict(config=True, help="Dict of Middleware") # Not sure if this should be optional. For now, make it configurable enable_custom_handlers = Bool(True, config=True, help="Enable Custom Handlers") root_dir = Unicode(getcwd()) def __init__(self, *args, **kwargs): super(MetaManager, self).__init__(*args, **kwargs) self.app = kwargs['parent'] self.managers = {} server_home = FileContentsManager() server_home.root_dir = self.root_dir self.managers['server-home'] = server_home if self.enable_custom_handlers: enable_custom_handlers() for alias, path in self.file_dirs.items(): fb = FileContentsManager() fb.root_dir = path self.managers[alias] = fb for alias, path in self.bundle_dirs.items(): fb = BundleNotebookManager() fb.root_dir = path self.managers[alias] = fb for user, pw in self.github_accounts: gh = notebook_gisthub(user, pw) gbm = GistNotebookManager(gisthub=gh) self.managers['gist:' + user] = gbm self.middleware = {} for name, middleware in self.manager_middleware.items(): cls = import_item(middleware) self.middleware[name] = cls(parent=self, log=self.log) self.root = RootManager(meta_manager=self) def dispatch_middleware(self, hook_name, *args, **kwargs): """ dispatch hook calls to middleware """ for name, middleware in self.middleware.items(): method = getattr(middleware, hook_name, None) if method is not None: method(*args, **kwargs) def _nbm_from_path(self, path='', name=None): """ So this helper function takes in a request path and returns the manager for that path. Note: I would like to only take in a single full request path, but IPython is odd in its handling of this. So I have to accept a name param. Sometimes the name is really a path. blah """ if self.debug: print('nbm_from_path(path={0}, name={1})'.format(path, name)) import inspect print(inspect.stack()[1][3]) # TODO clean up this logic. request_path = self._get_fullpath(name, path) # remove beginning slash (/) if request_path and request_path[0] == os.sep: request_path = request_path[1:] meta = ManagerMeta() # we are on root if not request_path: meta.request_path = '' meta.path = '' meta.name = '' return self.root, meta bits = request_path.split(os.sep) manager_path = bits.pop(0) # if we were passed in a name we should spit it back out # to remain compatible with IPython. This really only seems # to be an issue with get_model local_name = '' if name and len(bits) >= 1: local_name = bits.pop() local_path = os.sep.join(bits) meta.request_path = request_path meta.path = local_path meta.name = local_name meta.nbm_path = manager_path nbm = self.managers.get(manager_path) if self.debug: print(nbm, meta) return nbm, meta def _get_fullpath(self, name=None, path=''): if name is not None: path = url_path_join(path, name) return path def list_dirs(self, path): nbm, meta = self._nbm_from_path(path) val = nbm.list_dirs(meta.path) return val # ContentsManager API part 1 def path_exists(self, path): nbm, meta = self._nbm_from_path(path) if nbm is None: return False exists = nbm.path_exists(meta.path) return exists def is_hidden(self, path): nbm, meta = self._nbm_from_path(path) return nbm.is_hidden(meta.path) def file_exists(self, name, path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.file_exists(meta.name, meta.path) def exists(self, name, path=''): nbm, meta = self._nbm_from_path(path, name) if nbm is None: return False exists = nbm.exists(meta.name, meta.path) return exists def get_model(self, name, path='', content=True): nbm, meta = self._nbm_from_path(path, name) model = nbm.get_model(meta.name, path=meta.path, content=content) # while the local manager doesn't know its nbm_path, # we have to add it back in for the metamanager. if model['type'] == 'directory': content = model.get("content", []) for m in content: m['path'] = meta.request_path return model @manager_hook def save(self, model, name='', path=''): nbm, meta = self._nbm_from_path(path, name) # make sure path is local and doesn't include sub manager prefix model['path'] = meta.path model = nbm.save(model=model, name=meta.name, path=meta.path) return model def update(self, model, name, path=''): """Update the notebook and return the model with no content.""" nbm, meta = self._nbm_from_path(path, name) return nbm.update(model, meta.name, meta.path) def delete(self, name, path=''): """Delete notebook by name and path.""" nbm, meta = self._nbm_from_path(path, name) return nbm.delete(meta.name, meta.path) def create_checkpoint(self, name, path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.create_checkpoint(meta.name, meta.path) def list_checkpoints(self, name, path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.list_checkpoints(meta.name, meta.path) def restore_checkpoint(self, checkpoint_id, name, path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.restore_checkpoint(checkpoint_id, meta.name, meta.path) def delete_checkpoint(self, checkpoint_id, name, path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.delete_checkpoint(checkpoint_id, meta.name, meta.path) # ContentsManager API part 2: methods that have useable default # implementations, but can be overridden in subclasses. # Note, some of these just call the default ContentsManager # implementation. In reality, they won't need to ever be overridden. def info_string(self): infos = [nbm.info_string() for nbm in self.managers.values()] return "\n".join(infos) def get_kernel_path(self, name, path=''): """ defined where kernel for notebooks is started """ nbm, meta = self._nbm_from_path(path, name) return nbm.get_kernel_path(meta.name, meta.path) def increment_filename(self, filename, path=''): nbm, meta = self._nbm_from_path(path) return nbm.get_kernel_path(filename, meta.path) def create_file(self, model=None, path='', ext='.ipynb'): nbm, meta = self._nbm_from_path(path) return nbm.create_file(model, meta.path, ext) def copy(self, from_name, to_name=None, path=''): nbm, meta = self._nbm_from_path(path) model = nbm.copy(from_name, to_name, meta.path) model['path'] = path return model def trust_notebook(self, name, path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.trust_notebook(meta.name, meta.path) def check_and_sign(self, nb, name='', path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.check_and_sign(nb, meta.name, meta.path) def mark_trusted_cells(self, nb, name='', path=''): nbm, meta = self._nbm_from_path(path, name) return nbm.mark_trusted_cells(nb, meta.name, meta.path) ## END ContentsManager API # StaticFileHandler def get_absolute_path(self, path): """ return absolute path for static file handler """ nbm, meta = self._nbm_from_path(path) return nbm.get_absolute_path(meta.path)
class TemplateExporter(Exporter): """ Exports notebooks into other file formats. Uses Jinja 2 templating engine to output new formats. Inherit from this class if you are creating a new template type along with new filters/preprocessors. If the filters/ preprocessors provided by default suffice, there is no need to inherit from this class. Instead, override the template_file and file_extension traits via a config file. {filters} """ # finish the docstring __doc__ = __doc__.format(filters = '- '+'\n - '.join(default_filters.keys())) template_file = Unicode(u'default', config=True, help="Name of the template file to use") def _template_file_changed(self, name, old, new): if new == 'default': self.template_file = self.default_template else: self.template_file = new self.template = None self._load_template() default_template = Unicode(u'') template = Any() environment = Any() template_path = List(['.'], config=True) def _template_path_changed(self, name, old, new): self._load_template() default_template_path = Unicode( os.path.join("..", "templates"), help="Path where the template files are located.") template_skeleton_path = Unicode( os.path.join("..", "templates", "skeleton"), help="Path where the template skeleton files are located.") #Jinja block definitions jinja_comment_block_start = Unicode("", config=True) jinja_comment_block_end = Unicode("", config=True) jinja_variable_block_start = Unicode("", config=True) jinja_variable_block_end = Unicode("", config=True) jinja_logic_block_start = Unicode("", config=True) jinja_logic_block_end = Unicode("", config=True) #Extension that the template files use. template_extension = Unicode(".tpl", config=True) filters = Dict(config=True, help="""Dictionary of filters, by name and namespace, to add to the Jinja environment.""") raw_mimetypes = List(config=True, help="""formats of raw cells to be included in this Exporter's output.""" ) def _raw_mimetypes_default(self): return [self.output_mimetype, ''] def __init__(self, config=None, extra_loaders=None, **kw): """ Public constructor Parameters ---------- config : config User configuration instance. extra_loaders : list[of Jinja Loaders] ordered list of Jinja loader to find templates. Will be tried in order before the default FileSystem ones. template : str (optional, kw arg) Template to use when exporting. """ super(TemplateExporter, self).__init__(config=config, **kw) #Init self._init_template() self._init_environment(extra_loaders=extra_loaders) self._init_filters() def _load_template(self): """Load the Jinja template object from the template file This is a no-op if the template attribute is already defined, or the Jinja environment is not setup yet. This is triggered by various trait changes that would change the template. """ from jinja2 import TemplateNotFound if self.template is not None: return # called too early, do nothing if self.environment is None: return # Try different template names during conversion. First try to load the # template by name with extension added, then try loading the template # as if the name is explicitly specified, then try the name as a # 'flavor', and lastly just try to load the template by module name. try_names = [] if self.template_file: try_names.extend([ self.template_file + self.template_extension, self.template_file, ]) for try_name in try_names: self.log.debug("Attempting to load template %s", try_name) try: self.template = self.environment.get_template(try_name) except (TemplateNotFound, IOError): pass except Exception as e: self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True) else: self.log.info("Loaded template %s", try_name) break @docstring_nbformat_mod def from_notebook_node(self, nb, resources=None, **kw): """ Convert a notebook from a notebook node instance. Parameters ---------- nb : :class:`~{nbformat_mod}.nbbase.NotebookNode` Notebook node resources : dict Additional resources that can be accessed read/write by preprocessors and filters. """ nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw) resources.setdefault('raw_mimetypes', self.raw_mimetypes) self._load_template() if self.template is not None: output = self.template.render(nb=nb_copy, resources=resources) else: raise IOError('template file "%s" could not be found' % self.template_file) return output, resources def register_filter(self, name, jinja_filter): """ Register a filter. A filter is a function that accepts and acts on one string. The filters are accesible within the Jinja templating engine. Parameters ---------- name : str name to give the filter in the Jinja engine filter : filter """ if jinja_filter is None: raise TypeError('filter') isclass = isinstance(jinja_filter, type) constructed = not isclass #Handle filter's registration based on it's type if constructed and isinstance(jinja_filter, py3compat.string_types): #filter is a string, import the namespace and recursively call #this register_filter method filter_cls = import_item(jinja_filter) return self.register_filter(name, filter_cls) if constructed and hasattr(jinja_filter, '__call__'): #filter is a function, no need to construct it. self.environment.filters[name] = jinja_filter return jinja_filter elif isclass and isinstance(jinja_filter, MetaHasTraits): #filter is configurable. Make sure to pass in new default for #the enabled flag if one was specified. filter_instance = jinja_filter(parent=self) self.register_filter(name, filter_instance ) elif isclass: #filter is not configurable, construct it filter_instance = jinja_filter() self.register_filter(name, filter_instance) else: #filter is an instance of something without a __call__ #attribute. raise TypeError('filter') def _init_template(self): """ Make sure a template name is specified. If one isn't specified, try to build one from the information we know. """ self._template_file_changed('template_file', self.template_file, self.template_file) def _init_environment(self, extra_loaders=None): """ Create the Jinja templating environment. """ from jinja2 import Environment, ChoiceLoader, FileSystemLoader here = os.path.dirname(os.path.realpath(__file__)) loaders = [] if extra_loaders: loaders.extend(extra_loaders) paths = self.template_path paths.extend([os.path.join(here, self.default_template_path), os.path.join(here, self.template_skeleton_path)]) loaders.append(FileSystemLoader(paths)) self.environment = Environment( loader= ChoiceLoader(loaders), extensions=JINJA_EXTENSIONS ) #Set special Jinja2 syntax that will not conflict with latex. if self.jinja_logic_block_start: self.environment.block_start_string = self.jinja_logic_block_start if self.jinja_logic_block_end: self.environment.block_end_string = self.jinja_logic_block_end if self.jinja_variable_block_start: self.environment.variable_start_string = self.jinja_variable_block_start if self.jinja_variable_block_end: self.environment.variable_end_string = self.jinja_variable_block_end if self.jinja_comment_block_start: self.environment.comment_start_string = self.jinja_comment_block_start if self.jinja_comment_block_end: self.environment.comment_end_string = self.jinja_comment_block_end def _init_filters(self): """ Register all of the filters required for the exporter. """ #Add default filters to the Jinja2 environment for key, value in default_filters.items(): self.register_filter(key, value) #Load user filters. Overwrite existing filters if need be. if self.filters: for key, user_filter in self.filters.items(): self.register_filter(key, user_filter)
class BoundedIntTextWidget(_BoundedIntWidget): _view_name = Unicode('IntTextView', sync=True)
class IntSliderWidget(_BoundedIntWidget): _view_name = Unicode('IntSliderView', sync=True) orientation = Enum([u'horizontal', u'vertical'], u'horizontal', help="Vertical or horizontal.", sync=True)
class _IntWidget(DOMWidget): value = CInt(0, help="Int value", sync=True) disabled = Bool(False, help="Enable or disable user changes", sync=True) description = Unicode( help="Description of the value this widget represents", sync=True)
class IntTextWidget(_IntWidget): _view_name = Unicode('IntTextView', sync=True)
class FileNotebookManager(NotebookManager): save_script = Bool( False, config=True, help="""Automatically create a Python script when saving the notebook. For easier use of import, %run and %load across notebooks, a <notebook-name>.py script will be created next to any <notebook-name>.ipynb on each save. This can also be set with the short `--script` flag. """) checkpoint_dir = Unicode( config=True, help="""The location in which to keep notebook checkpoints By default, it is notebook-dir/.ipynb_checkpoints """) def _checkpoint_dir_default(self): return os.path.join(self.notebook_dir, '.ipynb_checkpoints') def _checkpoint_dir_changed(self, name, old, new): """do a bit of validation of the checkpoint dir""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. abs_new = os.path.abspath(new) self.checkpoint_dir = abs_new return if os.path.exists(new) and not os.path.isdir(new): raise TraitError("checkpoint dir %r is not a directory" % new) if not os.path.exists(new): self.log.info("Creating checkpoint dir %s", new) try: os.mkdir(new) except: raise TraitError("Couldn't create checkpoint dir %r" % new) def get_notebook_names(self, path=''): """List all notebook names in the notebook dir and path.""" path = path.strip('/') if not os.path.isdir(self.get_os_path(path=path)): raise web.HTTPError(404, 'Directory not found: ' + path) names = glob.glob(self.get_os_path('*' + self.filename_ext, path)) names = [os.path.basename(name) for name in names] return names def increment_filename(self, basename, path='', ext='.ipynb'): """Return a non-used filename of the form basename<int>.""" path = path.strip('/') for i in itertools.count(): name = u'{basename}{i}{ext}'.format(basename=basename, i=i, ext=ext) os_path = self.get_os_path(name, path) if not os.path.isfile(os_path): break return name def path_exists(self, path): """Does the API-style path (directory) actually exist? Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to base notebook-dir). Returns ------- exists : bool Whether the path is indeed a directory. """ path = path.strip('/') os_path = self.get_os_path(path=path) return os.path.isdir(os_path) def get_os_path(self, name=None, path=''): """Given a notebook name and a URL path, return its file system path. Parameters ---------- name : string The name of a notebook file with the .ipynb extension path : string The relative URL path (with '/' as separator) to the named notebook. Returns ------- path : string A file system path that combines notebook_dir (location where server started), the relative path, and the filename with the current operating system's url. """ parts = path.strip('/').split('/') parts = [p for p in parts if p != ''] # remove duplicate splits if name is not None: parts.append(name) path = os.path.join(self.notebook_dir, *parts) return path def notebook_exists(self, name, path=''): """Returns a True if the notebook exists. Else, returns False. Parameters ---------- name : string The name of the notebook you are checking. path : string The relative path to the notebook (with '/' as separator) Returns ------- bool """ path = path.strip('/') nbpath = self.get_os_path(name, path=path) return os.path.isfile(nbpath) def list_notebooks(self, path): """Returns a list of dictionaries that are the standard model for all notebooks in the relative 'path'. Parameters ---------- path : str the URL path that describes the relative path for the listed notebooks Returns ------- notebooks : list of dicts a list of the notebook models without 'content' """ path = path.strip('/') notebook_names = self.get_notebook_names(path) notebooks = [] for name in notebook_names: model = self.get_notebook_model(name, path, content=False) notebooks.append(model) notebooks = sorted(notebooks, key=lambda item: item['name']) return notebooks def get_notebook_model(self, name, path='', content=True): """ Takes a path and name for a notebook and returns it's model Parameters ---------- name : str the name of the notebook path : str the URL path that describes the relative path for the notebook Returns ------- model : dict the notebook model. If contents=True, returns the 'contents' dict in the model as well. """ path = path.strip('/') if not self.notebook_exists(name=name, path=path): raise web.HTTPError(404, u'Notebook does not exist: %s' % name) os_path = self.get_os_path(name, path) info = os.stat(os_path) last_modified = tz.utcfromtimestamp(info.st_mtime) created = tz.utcfromtimestamp(info.st_ctime) # Create the notebook model. model = {} model['name'] = name model['path'] = path model['last_modified'] = last_modified model['created'] = created if content is True: with io.open(os_path, 'r', encoding='utf-8') as f: try: nb = current.read(f, u'json') except Exception as e: raise web.HTTPError( 400, u"Unreadable Notebook: %s %s" % (os_path, e)) model['content'] = nb return model def save_notebook_model(self, model, name='', path=''): """Save the notebook model and return the model with no content.""" path = path.strip('/') if 'content' not in model: raise web.HTTPError(400, u'No notebook JSON data provided') # One checkpoint should always exist if self.notebook_exists( name, path) and not self.list_checkpoints(name, path): self.create_checkpoint(name, path) new_path = model.get('path', path).strip('/') new_name = model.get('name', name) if path != new_path or name != new_name: self.rename_notebook(name, path, new_name, new_path) # Save the notebook file os_path = self.get_os_path(new_name, new_path) nb = current.to_notebook_json(model['content']) if 'name' in nb['metadata']: nb['metadata']['name'] = u'' try: self.log.debug("Autosaving notebook %s", os_path) with io.open(os_path, 'w', encoding='utf-8') as f: current.write(nb, f, u'json') except Exception as e: raise web.HTTPError( 400, u'Unexpected error while autosaving notebook: %s %s' % (os_path, e)) # Save .py script as well if self.save_script: py_path = os.path.splitext(os_path)[0] + '.py' self.log.debug("Writing script %s", py_path) try: with io.open(py_path, 'w', encoding='utf-8') as f: current.write(nb, f, u'py') except Exception as e: raise web.HTTPError( 400, u'Unexpected error while saving notebook as script: %s %s' % (py_path, e)) model = self.get_notebook_model(new_name, new_path, content=False) return model def update_notebook_model(self, model, name, path=''): """Update the notebook's path and/or name""" path = path.strip('/') new_name = model.get('name', name) new_path = model.get('path', path).strip('/') if path != new_path or name != new_name: self.rename_notebook(name, path, new_name, new_path) model = self.get_notebook_model(new_name, new_path, content=False) return model def delete_notebook_model(self, name, path=''): """Delete notebook by name and path.""" path = path.strip('/') os_path = self.get_os_path(name, path) if not os.path.isfile(os_path): raise web.HTTPError(404, u'Notebook does not exist: %s' % os_path) # clear checkpoints for checkpoint in self.list_checkpoints(name, path): checkpoint_id = checkpoint['id'] cp_path = self.get_checkpoint_path(checkpoint_id, name, path) if os.path.isfile(cp_path): self.log.debug("Unlinking checkpoint %s", cp_path) os.unlink(cp_path) self.log.debug("Unlinking notebook %s", os_path) os.unlink(os_path) def rename_notebook(self, old_name, old_path, new_name, new_path): """Rename a notebook.""" old_path = old_path.strip('/') new_path = new_path.strip('/') if new_name == old_name and new_path == old_path: return new_os_path = self.get_os_path(new_name, new_path) old_os_path = self.get_os_path(old_name, old_path) # Should we proceed with the move? if os.path.isfile(new_os_path): raise web.HTTPError( 409, u'Notebook with name already exists: %s' % new_os_path) if self.save_script: old_py_path = os.path.splitext(old_os_path)[0] + '.py' new_py_path = os.path.splitext(new_os_path)[0] + '.py' if os.path.isfile(new_py_path): raise web.HTTPError( 409, u'Python script with name already exists: %s' % new_py_path) # Move the notebook file try: os.rename(old_os_path, new_os_path) except Exception as e: raise web.HTTPError( 500, u'Unknown error renaming notebook: %s %s' % (old_os_path, e)) # Move the checkpoints old_checkpoints = self.list_checkpoints(old_name, old_path) for cp in old_checkpoints: checkpoint_id = cp['id'] old_cp_path = self.get_checkpoint_path(checkpoint_id, old_name, old_path) new_cp_path = self.get_checkpoint_path(checkpoint_id, new_name, new_path) if os.path.isfile(old_cp_path): self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path) os.rename(old_cp_path, new_cp_path) # Move the .py script if self.save_script: os.rename(old_py_path, new_py_path) # Checkpoint-related utilities def get_checkpoint_path(self, checkpoint_id, name, path=''): """find the path to a checkpoint""" path = path.strip('/') basename, _ = os.path.splitext(name) filename = u"{name}-{checkpoint_id}{ext}".format( name=basename, checkpoint_id=checkpoint_id, ext=self.filename_ext, ) cp_path = os.path.join(path, self.checkpoint_dir, filename) return cp_path def get_checkpoint_model(self, checkpoint_id, name, path=''): """construct the info dict for a given checkpoint""" path = path.strip('/') cp_path = self.get_checkpoint_path(checkpoint_id, name, path) stats = os.stat(cp_path) last_modified = tz.utcfromtimestamp(stats.st_mtime) info = dict( id=checkpoint_id, last_modified=last_modified, ) return info # public checkpoint API def create_checkpoint(self, name, path=''): """Create a checkpoint from the current state of a notebook""" path = path.strip('/') nb_path = self.get_os_path(name, path) # only the one checkpoint ID: checkpoint_id = u"checkpoint" cp_path = self.get_checkpoint_path(checkpoint_id, name, path) self.log.debug("creating checkpoint for notebook %s", name) if not os.path.exists(self.checkpoint_dir): os.mkdir(self.checkpoint_dir) shutil.copy2(nb_path, cp_path) # return the checkpoint info return self.get_checkpoint_model(checkpoint_id, name, path) def list_checkpoints(self, name, path=''): """list the checkpoints for a given notebook This notebook manager currently only supports one checkpoint per notebook. """ path = path.strip('/') checkpoint_id = "checkpoint" path = self.get_checkpoint_path(checkpoint_id, name, path) if not os.path.exists(path): return [] else: return [self.get_checkpoint_model(checkpoint_id, name, path)] def restore_checkpoint(self, checkpoint_id, name, path=''): """restore a notebook to a checkpointed state""" path = path.strip('/') self.log.info("restoring Notebook %s from checkpoint %s", name, checkpoint_id) nb_path = self.get_os_path(name, path) cp_path = self.get_checkpoint_path(checkpoint_id, name, path) if not os.path.isfile(cp_path): self.log.debug("checkpoint file does not exist: %s", cp_path) raise web.HTTPError( 404, u'Notebook checkpoint does not exist: %s-%s' % (name, checkpoint_id)) # ensure notebook is readable (never restore from an unreadable notebook) with io.open(cp_path, 'r', encoding='utf-8') as f: nb = current.read(f, u'json') shutil.copy2(cp_path, nb_path) self.log.debug("copying %s -> %s", cp_path, nb_path) def delete_checkpoint(self, checkpoint_id, name, path=''): """delete a notebook's checkpoint""" path = path.strip('/') cp_path = self.get_checkpoint_path(checkpoint_id, name, path) if not os.path.isfile(cp_path): raise web.HTTPError( 404, u'Notebook checkpoint does not exist: %s%s-%s' % (path, name, checkpoint_id)) self.log.debug("unlinking %s", cp_path) os.unlink(cp_path) def info_string(self): return "Serving notebooks from local directory: %s" % self.notebook_dir
class Kernel(Configurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) def _eventloop_changed(self, name, old, new): """schedule call to eventloop from IOLoop""" loop = ioloop.IOLoop.instance() loop.add_callback(self.enter_eventloop) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') shell_class = Type(ZMQInteractiveShell) session = Instance(Session) profile_dir = Instance('IPython.core.profiledir.ProfileDir') shell_streams = List() control_stream = Instance(ZMQStream) iopub_socket = Instance(zmq.Socket) stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) user_module = Any() def _user_module_changed(self, name, old, new): if self.shell is not None: self.shell.user_module = new user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() # identities: int_id = Integer(-1) ident = Unicode() def _ident_default(self): return unicode_type(uuid.uuid4()) # Private interface _darwin_app_nap = Bool( True, config=True, help="""Whether to use appnope for compatiblity with OS X App Nap. Only affects OS X >= 10.9. """) # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005, config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05, config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() # A reference to the Python builtin 'raw_input' function. # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3) _sys_raw_input = Any() _sys_eval_input = Any() # set of aborted msg_ids aborted = Set() def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Initialize the InteractiveShell subclass self.shell = self.shell_class.instance( parent=self, profile_dir=self.profile_dir, user_module=self.user_module, user_ns=self.user_ns, kernel=self, ) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket self.shell.displayhook.topic = self._topic('pyout') self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket self.shell.data_pub.session = self.session self.shell.data_pub.pub_socket = self.iopub_socket # TMP - hack while developing self.shell._reply_content = None # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'object_info_request', 'history_request', 'kernel_info_request', 'connect_request', 'shutdown_request', 'apply_request', ] self.shell_handlers = {} for msg_type in msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) comm_msg_types = ['comm_open', 'comm_msg', 'comm_close'] comm_manager = self.shell.comm_manager for msg_type in comm_msg_types: self.shell_handlers[msg_type] = getattr(comm_manager, msg_type) control_msg_types = msg_types + ['clear_request', 'abort_request'] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) def dispatch_control(self, msg): """dispatch control requests""" idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Control Message", exc_info=True) return self.log.debug("Control received: %s", msg) header = msg['header'] msg_id = header['msg_id'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: try: handler(self.control_stream, idents, msg) except Exception: self.log.error("Exception in control handler:", exc_info=True) def dispatch_shell(self, stream, msg): """dispatch shell requests""" # flush control requests first if self.control_stream: self.control_stream.flush() idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) if msg_id in self.aborted: self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: # ensure default_int_handler during handler call sig = signal(SIGINT, default_int_handler) try: handler(stream, idents, msg) except Exception: self.log.error("Exception in message handler:", exc_info=True) finally: signal(SIGINT, sig) def enter_eventloop(self): """enter eventloop""" self.log.info("entering eventloop %s", self.eventloop) for stream in self.shell_streams: # flush any pending replies, # which may be skipped by entering the eventloop stream.flush(zmq.POLLOUT) # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel self.log.error("KeyboardInterrupt caught in kernel") continue else: # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break self.log.info("exiting eventloop") def start(self): """register dispatchers for streams""" self.shell.exit_now = False if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_shell(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) # publish idle status self._publish_status('starting') def do_one_iteration(self): """step eventloop just once""" if self.control_stream: self.control_stream.flush() for stream in self.shell_streams: # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _make_metadata(self, other=None): """init metadata dict, for execute/apply_reply""" new_md = { 'dependencies_met': True, 'engine': self.ident, 'started': datetime.now(), } if other: new_md.update(other) return new_md def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" self.session.send(self.iopub_socket, u'pyin', { u'code': code, u'execution_count': execution_count }, parent=parent, ident=self._topic('pyin')) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send( self.iopub_socket, u'status', {u'execution_state': status}, parent=parent, ident=self._topic('status'), ) def execute_request(self, stream, ident, parent): """handle an execute_request""" self._publish_status(u'busy', parent) try: content = parent[u'content'] code = py3compat.cast_unicode_py2(content[u'code']) silent = content[u'silent'] store_history = content.get(u'store_history', not silent) except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return md = self._make_metadata(parent['metadata']) shell = self.shell # we'll need this a lot here # Replace raw_input. Note that is not sufficient to replace # raw_input in the user namespace. if content.get('allow_stdin', False): raw_input = lambda prompt='': self._raw_input( prompt, ident, parent) input = lambda prompt='': eval(raw_input(prompt)) else: raw_input = input = lambda prompt='': self._no_raw_input() if py3compat.PY3: self._sys_raw_input = builtin_mod.input builtin_mod.input = raw_input else: self._sys_raw_input = builtin_mod.raw_input self._sys_eval_input = builtin_mod.input builtin_mod.raw_input = raw_input builtin_mod.input = input # Set the parent message of the display hook and out streams. shell.set_parent(parent) # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self._publish_pyin(code, parent, shell.execution_count) reply_content = {} try: # FIXME: the shell calls the exception handler itself. shell.run_cell(code, store_history=store_history, silent=silent) except: status = u'error' # FIXME: this code right now isn't being used yet by default, # because the run_cell() call above directly fires off exception # reporting. This code, therefore, is only active in the scenario # where runlines itself has an unhandled exception. We need to # uniformize this, for all exception construction to come from a # single location in the codbase. etype, evalue, tb = sys.exc_info() tb_list = traceback.format_exception(etype, evalue, tb) reply_content.update(shell._showtraceback(etype, evalue, tb_list)) else: status = u'ok' finally: # Restore raw_input. if py3compat.PY3: builtin_mod.input = self._sys_raw_input else: builtin_mod.raw_input = self._sys_raw_input builtin_mod.input = self._sys_eval_input reply_content[u'status'] = status # Return the execution counter so clients can display prompts reply_content['execution_count'] = shell.execution_count - 1 # FIXME - fish exception info out of shell, possibly left there by # runlines. We'll need to clean up this logic later. if shell._reply_content is not None: reply_content.update(shell._reply_content) e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute') reply_content['engine_info'] = e_info # reset after use shell._reply_content = None if 'traceback' in reply_content: self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback'])) # At this point, we can tell whether the main code execution succeeded # or not. If it did, we proceed to evaluate user_variables/expressions if reply_content['status'] == 'ok': reply_content[u'user_variables'] = \ shell.user_variables(content.get(u'user_variables', [])) reply_content[u'user_expressions'] = \ shell.user_expressions(content.get(u'user_expressions', {})) else: # If there was an error, don't even try to compute variables or # expressions reply_content[u'user_variables'] = {} reply_content[u'user_expressions'] = {} # Payloads should be retrieved regardless of outcome, so we can both # recover partial output (that could have been generated early in a # block, before an error) and clear the payload system always. reply_content[u'payload'] = shell.payload_manager.read_payload() # Be agressive about clearing the payload because we don't want # it to sit in memory until the next execute_request comes in. shell.payload_manager.clear_payload() # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) md['status'] = reply_content['status'] if reply_content['status'] == 'error' and \ reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=md, ident=ident) self.log.debug("%s", reply_msg) if not silent and reply_msg['content']['status'] == u'error': self._abort_queues() self._publish_status(u'idle', parent) def complete_request(self, stream, ident, parent): txt, matches = self._complete(parent) matches = {'matches': matches, 'matched_text': txt, 'status': 'ok'} matches = json_clean(matches) completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) def object_info_request(self, stream, ident, parent): content = parent['content'] object_info = self.shell.object_inspect(content['oname'], detail_level=content.get( 'detail_level', 0)) # Before we send this object over, we scrub it for JSON usage oinfo = json_clean(object_info) msg = self.session.send(stream, 'object_info_reply', oinfo, parent, ident) self.log.debug("%s", msg) def history_request(self, stream, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with # unicode keys before Python 2.6.5. hist_access_type = parent['content']['hist_access_type'] raw = parent['content']['raw'] output = parent['content']['output'] if hist_access_type == 'tail': n = parent['content']['n'] hist = self.shell.history_manager.get_tail(n, raw=raw, output=output, include_latest=True) elif hist_access_type == 'range': session = parent['content']['session'] start = parent['content']['start'] stop = parent['content']['stop'] hist = self.shell.history_manager.get_range(session, start, stop, raw=raw, output=output) elif hist_access_type == 'search': n = parent['content'].get('n') unique = parent['content'].get('unique', False) pattern = parent['content']['pattern'] hist = self.shell.history_manager.search(pattern, raw=raw, output=output, n=n, unique=unique) else: hist = [] hist = list(hist) content = {'history': hist} content = json_clean(content) msg = self.session.send(stream, 'history_reply', content, parent, ident) self.log.debug("Sending history reply with %i entries", len(hist)) def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) def kernel_info_request(self, stream, ident, parent): vinfo = { 'protocol_version': protocol_version, 'ipython_version': ipython_version, 'language_version': language_version, 'language': 'python', } msg = self.session.send(stream, 'kernel_info_reply', vinfo, parent, ident) self.log.debug("%s", msg) def shutdown_request(self, stream, ident, parent): self.shell.exit_now = True content = dict(status='ok') content.update(parent['content']) self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', content, parent) self._at_shutdown() # call sys.exit after a short delay loop = ioloop.IOLoop.instance() loop.add_timeout(time.time() + 0.1, loop.stop) #--------------------------------------------------------------------------- # Engine methods #--------------------------------------------------------------------------- def apply_request(self, stream, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] except: self.log.error("Got bad msg: %s", parent, exc_info=True) return self._publish_status(u'busy', parent) # Set the parent message of the display hook and out streams. shell = self.shell shell.set_parent(parent) # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_socket.send(pyin_msg) # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) md = self._make_metadata(parent['metadata']) try: working = shell.user_ns prefix = "_" + str(msg_id).replace("-", "") + "_" f, args, kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" ns = {fname: f, argname: args, kwargname: kwargs, resultname: None} # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec(code, shell.user_global_ns, shell.user_ns) result = working.get(resultname) finally: for key in ns: working.pop(key) result_buf = serialize_object( result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except: # invoke IPython traceback formatting shell.showtraceback() # FIXME - fish exception info out of shell, possibly left there by # run_code. We'll need to clean up this logic later. reply_content = {} if shell._reply_content is not None: reply_content.update(shell._reply_content) e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info # reset after use shell._reply_content = None self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent, ident=self._topic('pyerr')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] if reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False else: reply_content = {'status': 'ok'} # put 'ok'/'error' status in header, for scheduler introspection: md['status'] = reply_content['status'] # flush i/o sys.stdout.flush() sys.stderr.flush() reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident, buffers=result_buf, metadata=md) self._publish_status(u'idle', parent) #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- def abort_request(self, stream, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, string_types): msg_ids = [msg_ids] if not msg_ids: self.abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" self.shell.reset(False) msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content=dict(status='ok')) #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _wrap_exception(self, method=None): # import here, because _wrap_exception is only used in parallel, # and parallel has higher min pyzmq version from IPython.parallel.error import wrap_exception e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method) content = wrap_exception(e_info) return content def _topic(self, topic): """prefixed topic for IOPub messages""" if self.int_id >= 0: base = "engine.%i" % self.int_id else: base = "kernel.%s" % self.ident return py3compat.cast_bytes("%s.%s" % (base, topic)) def _abort_queues(self): for stream in self.shell_streams: if stream: self._abort_queue(stream) def _abort_queue(self, stream): poller = zmq.Poller() poller.register(stream.socket, zmq.POLLIN) while True: idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. poller.poll(50) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this " "frontend does not support stdin.") def _raw_input(self, prompt, ident, parent): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # flush the stdin socket, to purge stale replies while True: try: self.stdin_socket.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: break else: raise # Send the input request. content = json_clean(dict(prompt=prompt)) self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) except KeyboardInterrupt: # re-raise KeyboardInterrupt, to truncate traceback raise KeyboardInterrupt else: break try: value = py3compat.unicode_to_str(reply['content']['value']) except: self.log.error("Got bad raw_input reply: ") self.log.error("%s", parent) value = '' if value == '\x04': # EOF raise EOFError return value def _complete(self, msg): c = msg['content'] try: cpos = int(c['cursor_pos']) except: # If we don't get something that we can convert to an integer, at # least attempt the completion guessing the cursor is at the end of # the text, if there's any, and otherwise of the line cpos = len(c['text']) if cpos == 0: cpos = len(c['line']) return self.shell.complete(c['text'], c['line'], cpos) def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [s.flush(zmq.POLLOUT) for s in self.shell_streams]
class UnicodeTrait(HasTraits): value = Unicode(u'unicode')
class Exporter(Configurable): """ Exports notebooks into other file formats. Uses Jinja 2 templating engine to output new formats. Inherit from this class if you are creating a new template type along with new filters/transformers. If the filters/ transformers provided by default suffice, there is no need to inherit from this class. Instead, override the template_file and file_extension traits via a config file. {filters} """ # finish the docstring __doc__ = __doc__.format(filters='- ' + '\n - '.join(default_filters.keys())) template_file = Unicode('', config=True, help="Name of the template file to use") file_extension = Unicode( 'txt', config=True, help="Extension of the file that should be written to disk") template_path = Unicode("/../templates/", config=True, help="Path where the template files are located.") template_skeleton_path = Unicode( "/../templates/skeleton/", config=True, help="Path where the template skeleton files are located.") #Jinja block definitions jinja_comment_block_start = Unicode("", config=True) jinja_comment_block_end = Unicode("", config=True) jinja_variable_block_start = Unicode("", config=True) jinja_variable_block_end = Unicode("", config=True) jinja_logic_block_start = Unicode("", config=True) jinja_logic_block_end = Unicode("", config=True) #Extension that the template files use. template_extension = Unicode(".tpl", config=True) #Processors that process the input data prior to the export, set in the #constructor for this class. transformers = None _default_config = Config() def __init__(self, transformers=None, filters=None, config=None, **kw): """ Public constructor Parameters ---------- transformers : list[of transformer] Custom transformers to apply to the notebook prior to engaging the Jinja template engine. Any transformers specified here will override existing transformers if a naming conflict occurs. filters : dict[of filter] filters specified here will override existing filters if a naming conflict occurs. Filters are availlable in jinja template through the name of the corresponding key. Cf class docstring for availlable default filters. config : config User configuration instance. """ #Call the base class constructor super(Exporter, self).__init__(config=config, **kw) #Standard environment self._init_environment() #Add transformers self._register_transformers() #Add filters to the Jinja2 environment self._register_filters() #Load user transformers. Overwrite existing transformers if need be. if transformers: for transformer in transformers: self.register_transformer(transformer) #Load user filters. Overwrite existing filters if need be. if not filters is None: for key, user_filter in filters.iteritems(): if issubclass(user_filter, MetaHasTraits): self.environment.filters[key] = user_filter(config=config) else: self.environment.filters[key] = user_filter @property def default_config(self): if self._default_config: return Config(deepcopy(self._default_config)) else: return Config() def from_notebook_node(self, nb, resources=None): """ Convert a notebook from a notebook node instance. Parameters ---------- nb : Notebook node resources : a dict of additional resources that can be accessed read/write by transformers and filters. """ if resources is None: resources = {} nb, resources = self._preprocess(nb, resources) #Load the template file. self.template = self.environment.get_template(self.template_file + self.template_extension) return self.template.render(nb=nb, resources=resources), resources def from_filename(self, filename): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. """ with io.open(filename) as f: return self.from_notebook_node(nbformat.read(f, 'json')) def from_file(self, file_stream): """ Convert a notebook from a notebook file. Parameters ---------- file_stream : file-like object Notebook file-like object to convert. """ return self.from_notebook_node(nbformat.read(file_stream, 'json')) def register_transformer(self, transformer): """ Register a transformer. Transformers are classes that act upon the notebook before it is passed into the Jinja templating engine. Transformers are also capable of passing additional information to the Jinja templating engine. Parameters ---------- transformer : transformer """ if self.transformers is None: self.transformers = [] if inspect.isfunction(transformer): self.transformers.append(transformer) return transformer elif isinstance(transformer, MetaHasTraits): transformer_instance = transformer(config=self.config) self.transformers.append(transformer_instance) return transformer_instance else: transformer_instance = transformer() self.transformers.append(transformer_instance) return transformer_instance def register_filter(self, name, filter): """ Register a filter. A filter is a function that accepts and acts on one string. The filters are accesible within the Jinja templating engine. Parameters ---------- name : str name to give the filter in the Jinja engine filter : filter """ if inspect.isfunction(filter): self.environment.filters[name] = filter elif isinstance(filter, MetaHasTraits): self.environment.filters[name] = filter(config=self.config) else: self.environment.filters[name] = filter() return self.environment.filters[name] def _register_transformers(self): """ Register all of the transformers needed for this exporter. """ self.register_transformer(music21.ext.nbconvert.transformers. coalescestreams.coalesce_streams) #Remember the figure extraction transformer so it can be enabled and #disabled easily later. self.extract_figure_transformer = self.register_transformer( music21.ext.nbconvert.transformers.extractfigure. ExtractFigureTransformer) def _register_filters(self): """ Register all of the filters required for the exporter. """ for k, v in default_filters.iteritems(): self.register_filter(k, v) def _init_environment(self): """ Create the Jinja templating environment. """ self.environment = Environment(loader=FileSystemLoader([ os.path.dirname(os.path.realpath(__file__)) + self.template_path, os.path.dirname(os.path.realpath(__file__)) + self.template_skeleton_path, ]), extensions=JINJA_EXTENSIONS) #Set special Jinja2 syntax that will not conflict with latex. if self.jinja_logic_block_start: self.environment.block_start_string = self.jinja_logic_block_start if self.jinja_logic_block_end: self.environment.block_end_string = self.jinja_logic_block_end if self.jinja_variable_block_start: self.environment.variable_start_string = self.jinja_variable_block_start if self.jinja_variable_block_end: self.environment.variable_end_string = self.jinja_variable_block_end if self.jinja_comment_block_start: self.environment.comment_start_string = self.jinja_comment_block_start if self.jinja_comment_block_end: self.environment.comment_end_string = self.jinja_comment_block_end def _preprocess(self, nb, resources): """ Preprocess the notebook before passing it into the Jinja engine. To preprocess the notebook is to apply all of the Parameters ---------- nb : notebook node notebook that is being exported. resources : a dict of additional resources that can be accessed read/write by transformers and filters. """ # Do a deepcopy first, # we are never safe enough with what the transformers could do. nbc = deepcopy(nb) resc = deepcopy(resources) #Run each transformer on the notebook. Carry the output along #to each transformer for transformer in self.transformers: nb, resources = transformer(nbc, resc) return nb, resources
class IntProgressWidget(_BoundedIntWidget): _view_name = Unicode('ProgressView', sync=True)
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin): """ A Qt frontend for a generic Python kernel. """ # The text to show when the kernel is (re)started. banner = Unicode(config=True) # An option and corresponding signal for overriding the default kernel # interrupt behavior. custom_interrupt = Bool(False) custom_interrupt_requested = QtCore.Signal() # An option and corresponding signals for overriding the default kernel # restart behavior. custom_restart = Bool(False) custom_restart_kernel_died = QtCore.Signal(float) custom_restart_requested = QtCore.Signal() # Whether to automatically show calltips on open-parentheses. enable_calltips = Bool( True, config=True, help="Whether to draw information calltips on open-parentheses.") clear_on_kernel_restart = Bool( True, config=True, help="Whether to clear the console when the kernel is restarted") confirm_restart = Bool( True, config=True, help="Whether to ask for user confirmation when restarting kernel") # Emitted when a user visible 'execute_request' has been submitted to the # kernel from the FrontendWidget. Contains the code to be executed. executing = QtCore.Signal(object) # Emitted when a user-visible 'execute_reply' has been received from the # kernel and processed by the FrontendWidget. Contains the response message. executed = QtCore.Signal(object) # Emitted when an exit request has been received from the kernel. exit_requested = QtCore.Signal(object) # Protected class variables. _prompt_transformer = IPythonInputSplitter( physical_line_transforms=[classic_prompt()], logical_line_transforms=[], python_line_transforms=[], ) _CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos']) _CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos']) _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) _input_splitter_class = InputSplitter _local_kernel = False _highlighter = Instance(FrontendHighlighter) #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(FrontendWidget, self).__init__(*args, **kw) # FIXME: remove this when PySide min version is updated past 1.0.7 # forcefully disable calltips if PySide is < 1.0.7, because they crash if qt.QT_API == qt.QT_API_PYSIDE: import PySide if PySide.__version_info__ < (1, 0, 7): self.log.warn( "PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__) self.enable_calltips = False # FrontendWidget protected variables. self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._completion_lexer = CompletionLexer(PythonLexer()) self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None) self._hidden = False self._highlighter = FrontendHighlighter(self) self._input_splitter = self._input_splitter_class() self._kernel_manager = None self._kernel_client = None self._request_info = {} self._request_info['execute'] = {} self._callback_dict = {} # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Configure actions. action = self._copy_raw_action key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C action.setEnabled(False) action.setShortcut(QtGui.QKeySequence(key)) action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut) action.triggered.connect(self.copy_raw) self.copy_available.connect(action.setEnabled) self.addAction(action) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Set flag for whether we are connected via localhost. self._local_kernel = kw.get('local_kernel', FrontendWidget._local_kernel) #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- def copy(self): """ Copy the currently selected text to the clipboard, removing prompts. """ if self._page_control is not None and self._page_control.hasFocus(): self._page_control.copy() elif self._control.hasFocus(): text = self._control.textCursor().selection().toPlainText() if text: text = self._prompt_transformer.transform_cell(text) QtGui.QApplication.clipboard().setText(text) else: self.log.debug("frontend widget : unknown copy target") #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _is_complete(self, source, interactive): """ Returns whether 'source' can be completely processed and a new prompt created. When triggered by an Enter/Return key press, 'interactive' is True; otherwise, it is False. """ self._input_splitter.reset() complete = self._input_splitter.push(source) if interactive: complete = not self._input_splitter.push_accepts_more() return complete def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ msg_id = self.kernel_client.execute(source, hidden) self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'user') self._hidden = hidden if not hidden: self.executing.emit(source) def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ # Flush all state from the input splitter so the next round of # reading input starts with a clean buffer. self._input_splitter.reset() if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. text = self._get_input_buffer_cursor_line() if text is None: return False complete = bool(text[:self._get_input_buffer_cursor_column()].strip()) if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _context_menu_make(self, pos): """ Reimplemented to add an action for raw copy. """ menu = super(FrontendWidget, self)._context_menu_make(pos) for before_action in menu.actions(): if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \ QtGui.QKeySequence.ExactMatch: menu.insertAction(before_action, self._copy_raw_action) break return menu def request_interrupt_kernel(self): if self._executing: self.interrupt_kernel() def request_restart_kernel(self): message = 'Are you sure you want to restart the kernel?' self.restart_kernel(message, now=False) def _event_filter_console_keypress(self, event): """ Reimplemented for execution interruption and smart backspace. """ key = event.key() if self._control_key_down(event.modifiers(), include_command=False): if key == QtCore.Qt.Key_C and self._executing: self.request_interrupt_kernel() return True elif key == QtCore.Qt.Key_Period: self.request_restart_kernel() return True elif not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces if key == QtCore.Qt.Key_Backspace: col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(FrontendWidget, self)._event_filter_console_keypress(event) def _insert_continuation_prompt(self, cursor): """ Reimplemented for auto-indentation. """ super(FrontendWidget, self)._insert_continuation_prompt(cursor) cursor.insertText(' ' * self._input_splitter.indent_spaces) #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_complete_reply(self, rep): """ Handle replies for tab completion. """ self.log.debug("complete: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('complete') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): text = '.'.join(self._get_context()) cursor.movePosition(QtGui.QTextCursor.Left, n=len(text)) self._complete_with_items(cursor, rep['content']['matches']) def _silent_exec_callback(self, expr, callback): """Silently execute `expr` in the kernel and call `callback` with reply the `expr` is evaluated silently in the kernel (without) output in the frontend. Call `callback` with the `repr <http://docs.python.org/library/functions.html#repr> `_ as first argument Parameters ---------- expr : string valid string to be executed by the kernel. callback : function function accepting one argument, as a string. The string will be the `repr` of the result of evaluating `expr` The `callback` is called with the `repr()` of the result of `expr` as first argument. To get the object, do `eval()` on the passed value. See Also -------- _handle_exec_callback : private method, deal with calling callback with reply """ # generate uuid, which would be used as an indication of whether or # not the unique request originated from here (can use msg id ?) local_uuid = str(uuid.uuid1()) msg_id = self.kernel_client.execute( '', silent=True, user_expressions={local_uuid: expr}) self._callback_dict[local_uuid] = callback self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'silent_exec_callback') def _handle_exec_callback(self, msg): """Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback`` Parameters ---------- msg : raw message send by the kernel containing an `user_expressions` and having a 'silent_exec_callback' kind. Notes ----- This function will look for a `callback` associated with the corresponding message id. Association has been made by `_silent_exec_callback`. `callback` is then called with the `repr()` of the value of corresponding `user_expressions` as argument. `callback` is then removed from the known list so that any message coming again with the same id won't trigger it. """ user_exp = msg['content'].get('user_expressions') if not user_exp: return for expression in user_exp: if expression in self._callback_dict: self._callback_dict.pop(expression)(user_exp[expression]) def _handle_execute_reply(self, msg): """ Handles replies for code execution. """ self.log.debug("execute: %s", msg.get('content', '')) msg_id = msg['parent_header']['msg_id'] info = self._request_info['execute'].get(msg_id) # unset reading flag, because if execute finished, raw_input can't # still be pending. self._reading = False if info and info.kind == 'user' and not self._hidden: # Make sure that all output from the SUB channel has been processed # before writing a new prompt. self.kernel_client.iopub_channel.flush() # Reset the ANSI style information to prevent bad text in stdout # from messing up our colors. We're not a true terminal so we're # allowed to do this. if self.ansi_codes: self._ansi_processor.reset_sgr() content = msg['content'] status = content['status'] if status == 'ok': self._process_execute_ok(msg) elif status == 'error': self._process_execute_error(msg) elif status == 'aborted': self._process_execute_abort(msg) self._show_interpreter_prompt_for_reply(msg) self.executed.emit(msg) self._request_info['execute'].pop(msg_id) elif info and info.kind == 'silent_exec_callback' and not self._hidden: self._handle_exec_callback(msg) self._request_info['execute'].pop(msg_id) else: super(FrontendWidget, self)._handle_execute_reply(msg) def _handle_input_request(self, msg): """ Handle requests for raw_input. """ self.log.debug("input: %s", msg.get('content', '')) if self._hidden: raise RuntimeError( 'Request for raw input during hidden execution.') # Make sure that all output from the SUB channel has been processed # before entering readline mode. self.kernel_client.iopub_channel.flush() def callback(line): self.kernel_client.stdin_channel.input(line) if self._reading: self.log.debug( "Got second input request, assuming first was interrupted.") self._reading = False self._readline(msg['content']['prompt'], callback=callback) def _kernel_restarted_message(self, died=True): msg = "Kernel died, restarting" if died else "Kernel restarting" self._append_html("<br>%s<hr><br>" % msg, before_prompt=False) def _handle_kernel_died(self, since_last_heartbeat): """Handle the kernel's death (if we do not own the kernel). """ self.log.warn("kernel died: %s", since_last_heartbeat) if self.custom_restart: self.custom_restart_kernel_died.emit(since_last_heartbeat) else: self._kernel_restarted_message(died=True) self.reset() def _handle_kernel_restarted(self, died=True): """Notice that the autorestarter restarted the kernel. There's nothing to do but show a message. """ self.log.warn("kernel restarted") self._kernel_restarted_message(died=died) self.reset() def _handle_object_info_reply(self, rep): """ Handle replies for call tips. """ self.log.debug("oinfo: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('call_tip') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): # Get the information for a call tip. For now we format the call # line as string, later we can pass False to format_call and # syntax-highlight it ourselves for nicer formatting in the # calltip. content = rep['content'] # if this is from pykernel, 'docstring' will be the only key if content.get('ismagic', False): # Don't generate a call-tip for magics. Ideally, we should # generate a tooltip, but not on ( like we do for actual # callables. call_info, doc = None, None else: call_info, doc = call_tip(content, format_call=True) if call_info or doc: self._call_tip_widget.show_call_info(call_info, doc) def _handle_pyout(self, msg): """ Handle display hook output. """ self.log.debug("pyout: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): text = msg['content']['data'] self._append_plain_text(text + '\n', before_prompt=True) def _handle_stream(self, msg): """ Handle stdout, stderr, and stdin. """ self.log.debug("stream: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): # Most consoles treat tabs as being 8 space characters. Convert tabs # to spaces so that output looks as expected regardless of this # widget's tab width. text = msg['content']['data'].expandtabs(8) self._append_plain_text(text, before_prompt=True) self._control.moveCursor(QtGui.QTextCursor.End) def _handle_shutdown_reply(self, msg): """ Handle shutdown signal, only if from other console. """ self.log.warn("shutdown: %s", msg.get('content', '')) restart = msg.get('content', {}).get('restart', False) if not self._hidden and not self._is_from_this_session(msg): # got shutdown reply, request came from session other than ours if restart: # someone restarted the kernel, handle it self._handle_kernel_restarted(died=False) else: # kernel was shutdown permanently # this triggers exit_requested if the kernel was local, # and a dialog if the kernel was remote, # so we don't suddenly clear the qtconsole without asking. if self._local_kernel: self.exit_requested.emit(self) else: title = self.window().windowTitle() reply = QtGui.QMessageBox.question( self, title, "Kernel has been shutdown permanently. " "Close the Console?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.Yes: self.exit_requested.emit(self) def _handle_status(self, msg): """Handle status message""" # This is where a busy/idle indicator would be triggered, # when we make one. state = msg['content'].get('execution_state', '') if state == 'starting': # kernel started while we were running if self._executing: self._handle_kernel_restarted(died=True) elif state == 'idle': pass elif state == 'busy': pass def _started_channels(self): """ Called when the KernelManager channels have started listening or when the frontend is assigned an already listening KernelManager. """ self.reset(clear=True) #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def copy_raw(self): """ Copy the currently selected text to the clipboard without attempting to remove prompts or otherwise alter the text. """ self._control.copy() def execute_file(self, path, hidden=False): """ Attempts to execute file with 'path'. If 'hidden', no output is shown. """ self.execute('execfile(%r)' % path, hidden=hidden) def interrupt_kernel(self): """ Attempts to interrupt the running kernel. Also unsets _reading flag, to avoid runtime errors if raw_input is called again. """ if self.custom_interrupt: self._reading = False self.custom_interrupt_requested.emit() elif self.kernel_manager: self._reading = False self.kernel_manager.interrupt_kernel() else: self._append_plain_text( 'Cannot interrupt a kernel I did not start.\n') def reset(self, clear=False): """ Resets the widget to its initial state if ``clear`` parameter is True, otherwise prints a visual indication of the fact that the kernel restarted, but does not clear the traces from previous usage of the kernel before it was restarted. With ``clear=True``, it is similar to ``%clear``, but also re-writes the banner and aborts execution if necessary. """ if self._executing: self._executing = False self._request_info['execute'] = {} self._reading = False self._highlighter.highlighting_on = False if clear: self._control.clear() self._append_plain_text(self.banner) # update output marker for stdout/stderr, so that startup # messages appear after banner: self._append_before_prompt_pos = self._get_cursor().position() self._show_interpreter_prompt() def restart_kernel(self, message, now=False): """ Attempts to restart the running kernel. """ # FIXME: now should be configurable via a checkbox in the dialog. Right # now at least the heartbeat path sets it to True and the manual restart # to False. But those should just be the pre-selected states of a # checkbox that the user could override if so desired. But I don't know # enough Qt to go implementing the checkbox now. if self.custom_restart: self.custom_restart_requested.emit() return if self.kernel_manager: # Pause the heart beat channel to prevent further warnings. self.kernel_client.hb_channel.pause() # Prompt the user to restart the kernel. Un-pause the heartbeat if # they decline. (If they accept, the heartbeat will be un-paused # automatically when the kernel is restarted.) if self.confirm_restart: buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No result = QtGui.QMessageBox.question(self, 'Restart kernel?', message, buttons) do_restart = result == QtGui.QMessageBox.Yes else: # confirm_restart is False, so we don't need to ask user # anything, just do the restart do_restart = True if do_restart: try: self.kernel_manager.restart_kernel(now=now) except RuntimeError as e: self._append_plain_text('Error restarting kernel: %s\n' % e, before_prompt=True) else: self._append_html( "<br>Restarting kernel...\n<hr><br>", before_prompt=True, ) else: self.kernel_client.hb_channel.unpause() else: self._append_plain_text( 'Cannot restart a Kernel I did not start\n', before_prompt=True) #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _call_tip(self): """ Shows a call tip, if appropriate, at the current cursor location. """ # Decide if it makes sense to show a call tip if not self.enable_calltips: return False cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) != '(': return False context = self._get_context(cursor) if not context: return False # Send the metadata request to the kernel name = '.'.join(context) msg_id = self.kernel_client.object_info(name) pos = self._get_cursor().position() self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) return True def _complete(self): """ Performs completion at the current cursor location. """ context = self._get_context() if context: # Send the completion request to the kernel msg_id = self.kernel_client.complete( '.'.join(context), # text self._get_input_buffer_cursor_line(), # line self._get_input_buffer_cursor_column(), # cursor_pos self.input_buffer) # block pos = self._get_cursor().position() info = self._CompletionRequest(msg_id, pos) self._request_info['complete'] = info def _get_context(self, cursor=None): """ Gets the context for the specified cursor (or the current cursor if none is specified). """ if cursor is None: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.StartOfBlock, QtGui.QTextCursor.KeepAnchor) text = cursor.selection().toPlainText() return self._completion_lexer.get_context(text) def _process_execute_abort(self, msg): """ Process a reply for an aborted execution request. """ self._append_plain_text("ERROR: execution aborted\n") def _process_execute_error(self, msg): """ Process a reply for an execution request that resulted in an error. """ content = msg['content'] # If a SystemExit is passed along, this means exit() was called - also # all the ipython %exit magic syntax of '-k' to be used to keep # the kernel running if content['ename'] == 'SystemExit': keepkernel = content['evalue'] == '-k' or content[ 'evalue'] == 'True' self._keep_kernel_on_exit = keepkernel self.exit_requested.emit(self) else: traceback = ''.join(content['traceback']) self._append_plain_text(traceback) def _process_execute_ok(self, msg): """ Process a reply for a successful execution request. """ payload = msg['content']['payload'] for item in payload: if not self._process_execute_payload(item): warning = 'Warning: received unknown payload of type %s' print(warning % repr(item['source'])) def _process_execute_payload(self, item): """ Process a single payload item from the list of payload items in an execution reply. Returns whether the payload was handled. """ # The basic FrontendWidget doesn't handle payloads, as they are a # mechanism for going beyond the standard Python interpreter model. return False def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self._show_prompt('>>> ') def _show_interpreter_prompt_for_reply(self, msg): """ Shows a prompt for the interpreter given an 'execute_reply' message. """ self._show_interpreter_prompt() #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._call_tip() #------ Trait default initializers ----------------------------------------- def _banner_default(self): """ Returns the standard Python banner. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform)
class Foo(Configurable): i = Integer(0, config=True, help="The integer i.") j = Integer(1, config=True, help="The integer j.") name = Unicode(u'Brian', config=True, help="First name.")
class ProfileList(Application): name = 'ipython-profile' description = list_help examples = _list_examples aliases = Dict({ 'ipython-dir': 'ProfileList.ipython_dir', 'log-level': 'Application.log_level', }) flags = Dict( dict(debug=({ 'Application': { 'log_level': 0 } }, "Set Application.log_level to 0, maximizing log output."))) ipython_dir = Unicode(get_ipython_dir(), config=True, help=""" The name of the IPython directory. This directory is used for logging configuration (through profiles), history storage, etc. The default is usually $HOME/.ipython. This options can also be specified through the environment variable IPYTHONDIR. """) def _print_profiles(self, profiles): """print list of profiles, indented.""" for profile in profiles: print(' %s' % profile) def list_profile_dirs(self): profiles = list_bundled_profiles() if profiles: print() print("Available profiles in IPython:") self._print_profiles(profiles) print() print(" The first request for a bundled profile will copy it") print(" into your IPython directory (%s)," % self.ipython_dir) print(" where you can customize it.") profiles = list_profiles_in(self.ipython_dir) if profiles: print() print("Available profiles in %s:" % self.ipython_dir) self._print_profiles(profiles) profiles = list_profiles_in(os.getcwd()) if profiles: print() print("Available profiles in current directory (%s):" % os.getcwd()) self._print_profiles(profiles) print() print("To use any of the above profiles, start IPython with:") print(" ipython --profile=<name>") print() def start(self): self.list_profile_dirs()
class Comm(LoggingConfigurable): # If this is instantiated by a non-IPython kernel, shell will be None shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) kernel = Instance('IPython.kernel.zmq.kernelbase.Kernel') def _kernel_default(self): if Kernel.initialized(): return Kernel.instance() iopub_socket = Any() def _iopub_socket_default(self): return self.kernel.iopub_socket session = Instance('IPython.kernel.zmq.session.Session') def _session_default(self): if self.kernel is not None: return self.kernel.session target_name = Unicode('comm') topic = Bytes() def _topic_default(self): return ('comm-%s' % self.comm_id).encode('ascii') _open_data = Dict(help="data dict, if any, to be included in comm_open") _close_data = Dict(help="data dict, if any, to be included in comm_close") _msg_callback = Any() _close_callback = Any() _closed = Bool(False) comm_id = Unicode() def _comm_id_default(self): return uuid.uuid4().hex primary = Bool(True, help="Am I the primary or secondary Comm?") def __init__(self, target_name='', data=None, **kwargs): if target_name: kwargs['target_name'] = target_name super(Comm, self).__init__(**kwargs) if self.primary: # I am primary, open my peer. self.open(data) def _publish_msg(self, msg_type, data=None, metadata=None, buffers=None, **keys): """Helper for sending a comm message on IOPub""" data = {} if data is None else data metadata = {} if metadata is None else metadata content = json_clean(dict(data=data, comm_id=self.comm_id, **keys)) self.session.send( self.iopub_socket, msg_type, content, metadata=json_clean(metadata), parent=self.kernel._parent_header, ident=self.topic, buffers=buffers, ) def __del__(self): """trigger close on gc""" self.close() # publishing messages def open(self, data=None, metadata=None, buffers=None): """Open the frontend-side version of this comm""" if data is None: data = self._open_data comm_manager = getattr(self.kernel, 'comm_manager', None) if comm_manager is None: raise RuntimeError("Comms cannot be opened without a kernel " "and a comm_manager attached to that kernel.") comm_manager.register_comm(self) self._closed = False self._publish_msg( 'comm_open', data=data, metadata=metadata, buffers=buffers, target_name=self.target_name, ) def close(self, data=None, metadata=None, buffers=None): """Close the frontend-side version of this comm""" if self._closed: # only close once return if data is None: data = self._close_data self._publish_msg( 'comm_close', data=data, metadata=metadata, buffers=buffers, ) self.kernel.comm_manager.unregister_comm(self) self._closed = True def send(self, data=None, metadata=None, buffers=None): """Send a message to the frontend-side version of this comm""" self._publish_msg( 'comm_msg', data=data, metadata=metadata, buffers=buffers, ) # registering callbacks def on_close(self, callback): """Register a callback for comm_close Will be called with the `data` of the close message. Call `on_close(None)` to disable an existing callback. """ self._close_callback = callback def on_msg(self, callback): """Register a callback for comm_msg Will be called with the `data` of any comm_msg messages. Call `on_msg(None)` to disable an existing callback. """ self._msg_callback = callback # handling of incoming messages def handle_close(self, msg): """Handle a comm_close message""" self.log.debug("handle_close[%s](%s)", self.comm_id, msg) if self._close_callback: self._close_callback(msg) def handle_msg(self, msg): """Handle a comm_msg message""" self.log.debug("handle_msg[%s](%s)", self.comm_id, msg) if self._msg_callback: if self.shell: self.shell.events.trigger('pre_execute') self._msg_callback(msg) if self.shell: self.shell.events.trigger('post_execute')
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples aliases = aliases flags = flags classes = [ KernelManager, ProfileDir, Session, MappingKernelManager, ContentsManager, FileContentsManager, NotebookNotary, ] flags = Dict(flags) aliases = Dict(aliases) subcommands = dict(list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), ) ipython_kernel_argv = List(Unicode) _log_formatter_cls = LogFormatter def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%H:%M:%S" def _log_format_default(self): """override default log format to include time""" return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('', config=True) # Network related information allow_origin = Unicode('', config=True, help="""Set the Access-Control-Allow-Origin header Use '*' to allow any origin to access your server. Takes precedence over allow_origin_pat. """) allow_origin_pat = Unicode( '', config=True, help= """Use a regular expression for the Access-Control-Allow-Origin header Requests from an origin matching the expression will get replies with: Access-Control-Allow-Origin: origin where `origin` is the origin of the request. Ignored if allow_origin is set. """) allow_credentials = Bool( False, config=True, help="Set the Access-Control-Allow-Credentials: true header") default_url = Unicode('/tree', config=True, help="The default URL to redirect to from `/`") ip = Unicode('localhost', config=True, help="The IP address the notebook server will listen on.") def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on.") port_retries = Integer( 50, config=True, help= "The number of additional ports to try if the specified port is not available." ) certfile = Unicode( u'', config=True, help="""The full path to an SSL/TLS certificate file.""") keyfile = Unicode( u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""") cookie_secret_file = Unicode( config=True, help="""The file where the cookie secret is stored.""") def _cookie_secret_file_default(self): if self.profile_dir is None: return '' return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret') cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """) def _cookie_secret_default(self): if os.path.exists(self.cookie_secret_file): with io.open(self.cookie_secret_file, 'rb') as f: return f.read() else: secret = base64.encodestring(os.urandom(1024)) self._write_cookie_secret_file(secret) return secret def _write_cookie_secret_file(self, secret): """write my secret to my secret_file""" self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file) with io.open(self.cookie_secret_file, 'wb') as f: f.write(secret) try: os.chmod(self.cookie_secret_file, 0o600) except OSError: self.log.warn("Could not set permissions on %s", self.cookie_secret_file) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict(config=True, help="DEPRECATED, use tornado_settings") def _webapp_settings_changed(self, name, old, new): self.log.warn( "\n webapp_settings is deprecated, use tornado_settings.\n") self.tornado_settings = new tornado_settings = Dict( config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") jinja_environment_options = Dict( config=True, help="Supply extra arguments that will be passed to Jinja environment." ) enable_mathjax = Bool( True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_url_changed(self, name, old, new): if not new.startswith('/'): self.base_url = '/' + new elif not new.endswith('/'): self.base_url = new + '/' base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""") def _base_project_url_changed(self, name, old, new): self.log.warn("base_project_url is deprecated, use base_url") self.base_url = new extra_static_paths = List( Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""") def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] extra_template_paths = List( Unicode, config=True, help="""Extra paths to search for serving jinja templates. Can be used to override templates from IPython.html.templates.""") def _extra_template_paths_default(self): return [] @property def template_file_path(self): """return extra paths + the default locations""" return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST extra_nbextensions_path = List( Unicode, config=True, help="""extra paths to look for Javascript notebook extensions""") @property def nbextensions_path(self): """The path to look for Javascript notebook extensions""" return self.extra_nbextensions_path + [ os.path.join(get_ipython_dir(), 'nbextensions') ] + SYSTEM_NBEXTENSIONS_DIRS websocket_url = Unicode("", config=True, help="""The base URL for websockets, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """) mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""") def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.tornado_settings.get( "static_url_prefix", url_path_join(self.base_url, "static")) # try local mathjax, either in nbextensions/mathjax or static/mathjax for (url_prefix, search_path) in [ (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path), (static_url_prefix, self.static_file_path), ]: self.log.debug("searching for local mathjax in %s", search_path) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path) except IOError: continue else: url = url_path_join(url_prefix, u"mathjax/MathJax.js") self.log.info("Serving local MathJax from %s at %s", mathjax, url) return url # no local mathjax, serve from CDN url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) contents_manager_class = Type(default_value=FileContentsManager, klass=ContentsManager, config=True, help='The notebook manager class to use.') kernel_manager_class = Type(default_value=MappingKernelManager, config=True, help='The kernel manager class to use.') session_manager_class = Type(default_value=SessionManager, config=True, help='The session manager class to use.') cluster_manager_class = Type(default_value=ClusterManager, config=True, help='The cluster manager class to use.') config_manager_class = Type(default_value=ConfigManager, config=True, help='The config manager class to use') kernel_spec_manager = Instance(KernelSpecManager) def _kernel_spec_manager_default(self): return KernelSpecManager(ipython_dir=self.ipython_dir) kernel_spec_manager_class = Type(default_value=KernelSpecManager, config=True, help=""" The kernel spec manager class to use. Should be a subclass of `IPython.kernel.kernelspec.KernelSpecManager`. The Api of KernelSpecManager is provisional and might change without warning between this version of IPython and the next stable one. """) login_handler_class = Type( default_value=LoginHandler, klass=web.RequestHandler, config=True, help='The login handler class to use.', ) logout_handler_class = Type( default_value=LogoutHandler, klass=web.RequestHandler, config=True, help='The logout handler class to use.', ) trust_xheaders = Bool( False, config=True, help= ("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL" )) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json" % os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) pylab = Unicode('disabled', config=True, help=""" DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. """) def _pylab_changed(self, name, old, new): """when --pylab is specified, display a warning and exit""" if new != 'warn': backend = ' %s' % new else: backend = '' self.log.error( "Support for specifying --pylab on the command line has been removed." ) self.log.error( "Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself." .format(backend)) self.exit(1) notebook_dir = Unicode( config=True, help="The directory to use for notebooks and kernels.") def _notebook_dir_default(self): if self.file_to_run: return os.path.dirname(os.path.abspath(self.file_to_run)) else: return py3compat.getcwd() def _notebook_dir_changed(self, name, old, new): """Do a bit of validation of the notebook dir.""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. self.notebook_dir = os.path.abspath(new) return if not os.path.isdir(new): raise TraitError("No such notebook dir: %r" % new) # setting App.notebook_dir implies setting notebook and kernel dirs as well self.config.FileContentsManager.root_dir = new self.config.MappingKernelManager.root_dir = new server_extensions = List( Unicode(), config=True, help=( "Python modules to load as notebook server extensions. " "This is an experimental API, and may change in future releases.")) def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: arg0 = self.extra_args[0] f = os.path.abspath(arg0) self.argv.remove(arg0) if not os.path.exists(f): self.log.critical("No such file or directory: %s", f) self.exit(1) # Use config here, to ensure that it takes higher priority than # anything that comes from the profile. c = Config() if os.path.isdir(f): c.NotebookApp.notebook_dir = f elif os.path.isfile(f): c.NotebookApp.file_to_run = f self.update_config(c) def init_kernel_argv(self): """add the profile-dir to arguments to be passed to IPython kernels""" # FIXME: remove special treatment of IPython kernels # Kernel should get *absolute* path to profile directory self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location] def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_spec_manager = self.kernel_spec_manager_class( ipython_dir=self.ipython_dir, ) self.kernel_manager = self.kernel_manager_class( parent=self, log=self.log, ipython_kernel_argv=self.ipython_kernel_argv, connection_dir=self.profile_dir.security_dir, ) self.contents_manager = self.contents_manager_class( parent=self, log=self.log, ) self.session_manager = self.session_manager_class( parent=self, log=self.log, kernel_manager=self.kernel_manager, contents_manager=self.contents_manager, ) self.cluster_manager = self.cluster_manager_class( parent=self, log=self.log, ) self.config_manager = self.config_manager_class( parent=self, log=self.log, profile_dir=self.profile_dir.location, ) def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False for log in app_log, access_log, gen_log: # consistent log output name (NotebookApp instead of tornado.access, etc.) log.name = self.log.name # hook up tornado 3's loggers to our app handlers logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.tornado_settings['allow_origin'] = self.allow_origin if self.allow_origin_pat: self.tornado_settings['allow_origin_pat'] = re.compile( self.allow_origin_pat) self.tornado_settings['allow_credentials'] = self.allow_credentials self.web_app = NotebookWebApplication( self, self.kernel_manager, self.contents_manager, self.cluster_manager, self.session_manager, self.kernel_spec_manager, self.config_manager, self.log, self.base_url, self.default_url, self.tornado_settings, self.jinja_environment_options) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.login_handler_class.validate_security(self, ssl_options=ssl_options) self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) success = None for port in random_ports(self.port, self.port_retries + 1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info( 'The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical( 'ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else 'localhost' return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) def init_terminals(self): try: from .terminal import initialize initialize(self.web_app) self.web_app.settings['terminals_available'] = True except ImportError as e: log = self.log.debug if sys.platform == 'win32' else self.log.warn log("Terminals not available (error was %s)", e) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r, w, x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y') and 'n' not in line.lower(): self.log.critical("Shutdown confirmed") ioloop.IOLoop.current().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.current().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.current().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn( "components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn( "components submodule unclean, you may see 404s on static/components" ) self.log.warn( "run `setup.py submodule` or `git submodule update` to update") def init_server_extensions(self): """Load any extensions specified by config. Import the module, then call the load_jupyter_server_extension function, if one exists. The extension API is experimental, and may change in future releases. """ for modulename in self.server_extensions: try: mod = importlib.import_module(modulename) func = getattr(mod, 'load_jupyter_server_extension', None) if func is not None: func(self) except Exception: self.log.warn("Error loading server extension %s", modulename, exc_info=True) @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_terminals() self.init_signal() self.init_server_extensions() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.contents_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return { 'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), 'base_url': self.base_url, 'notebook_dir': os.path.abspath(self.notebook_dir), 'pid': os.getpid() } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: if not os.path.exists(self.file_to_run): self.log.critical("%s does not exist" % self.file_to_run) self.exit(1) relpath = os.path.relpath(self.file_to_run, self.notebook_dir) uri = url_path_join('notebooks', *relpath.split(os.sep)) else: uri = 'tree' if browser: b = lambda: browser.open( url_path_join(self.connection_url, uri), new=2) threading.Thread(target=b).start() self.io_loop = ioloop.IOLoop.current() try: self.io_loop.start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file() def stop(self): def _stop(): self.http_server.stop() self.io_loop.stop() self.io_loop.add_callback(_stop)
class BaseDB(LoggingConfigurable): """Empty Parent class so traitlets work on DB.""" # base configurable traits: session = Unicode("")
class InlineBackend(InlineBackendConfig): """An object to store configuration of the inline backend.""" def _config_changed(self, name, old, new): # warn on change of renamed config section if new.InlineBackendConfig != getattr(old, 'InlineBackendConfig', Config()): warn("InlineBackendConfig has been renamed to InlineBackend") super(InlineBackend, self)._config_changed(name, old, new) # The typical default figure size is too large for inline use, # so we shrink the figure size to 6x4, and tweak fonts to # make that fit. rc = Dict( { 'figure.figsize': (6.0, 4.0), # play nicely with white background in the Qt and notebook frontend 'figure.facecolor': (1, 1, 1, 0), 'figure.edgecolor': (1, 1, 1, 0), 'axes.facecolor': (1, 1, 1, 0), # 12pt labels get cutoff on 6x4 logplots, so use 10pt. 'font.size': 10, # 72 dpi matches SVG/qtconsole # this only affects PNG export, as SVG has no dpi setting 'savefig.dpi': 72, # 10pt still needs a little more room on the xlabel: 'figure.subplot.bottom': .125 }, config=True, help="""Subset of matplotlib rcParams that should be different for the inline backend.""") figure_formats = Set({'png'}, config=True, help="""A set of figure formats to enable: 'png', 'retina', 'jpeg', 'svg', 'pdf'.""") def _update_figure_formatters(self): if self.shell is not None: from IPython.core.pylabtools import select_figure_formats select_figure_formats(self.shell, self.figure_formats, **self.print_figure_kwargs) def _figure_formats_changed(self, name, old, new): if 'jpg' in new or 'jpeg' in new: if not pil_available(): raise TraitError("Requires PIL/Pillow for JPG figures") self._update_figure_formatters() figure_format = Unicode(config=True, help="""The figure format to enable (deprecated use `figure_formats` instead)""") def _figure_format_changed(self, name, old, new): if new: self.figure_formats = {new} print_figure_kwargs = Dict( {'bbox_inches': 'tight'}, config=True, help="""Extra kwargs to be passed to fig.canvas.print_figure. Logical examples include: bbox_inches, quality (for jpeg figures), etc. """) _print_figure_kwargs_changed = _update_figure_formatters close_figures = Bool(True, config=True, help="""Close all figures at the end of each cell. When True, ensures that each cell starts with no active figures, but it also means that one must keep track of references in order to edit or redraw figures in subsequent cells. This mode is ideal for the notebook, where residual plots from other cells might be surprising. When False, one must call figure() to create new figures. This means that gcf() and getfigs() can reference figures created in other cells, and the active figure can continue to be edited with pylab/pyplot methods that reference the current active figure. This mode facilitates iterative editing of figures, and behaves most consistently with other matplotlib backends, but figure barriers between cells must be explicit. """) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
class Kernel(SingletonConfigurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) def _eventloop_changed(self, name, old, new): """schedule call to eventloop from IOLoop""" loop = ioloop.IOLoop.instance() loop.add_callback(self.enter_eventloop) session = Instance(Session, allow_none=True) profile_dir = Instance('IPython.core.profiledir.ProfileDir', allow_none=True) shell_streams = List() control_stream = Instance(ZMQStream, allow_none=True) iopub_socket = Instance(zmq.Socket, allow_none=True) stdin_socket = Instance(zmq.Socket, allow_none=True) log = Instance(logging.Logger, allow_none=True) # identities: int_id = Integer(-1) ident = Unicode() def _ident_default(self): return unicode_type(uuid.uuid4()) # This should be overridden by wrapper kernels that implement any real # language. language_info = {} # any links that should go in the help menu help_links = List() # Private interface _darwin_app_nap = Bool( True, config=True, help="""Whether to use appnope for compatiblity with OS X App Nap. Only affects OS X >= 10.9. """) # track associations with current request _allow_stdin = Bool(False) _parent_header = Dict() _parent_ident = Any(b'') # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005, config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05, config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() # set of aborted msg_ids aborted = Set() # Track execution count here. For IPython, we override this to use the # execution count we store in the shell. execution_count = 0 def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'inspect_request', 'history_request', 'kernel_info_request', 'connect_request', 'shutdown_request', 'apply_request', 'is_complete_request', ] self.shell_handlers = {} for msg_type in msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) control_msg_types = msg_types + ['clear_request', 'abort_request'] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) def dispatch_control(self, msg): """dispatch control requests""" idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.deserialize(msg, content=True, copy=False) except: self.log.error("Invalid Control Message", exc_info=True) return self.log.debug("Control received: %s", msg) # Set the parent message for side effects. self.set_parent(idents, msg) self._publish_status(u'busy') header = msg['header'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: try: handler(self.control_stream, idents, msg) except Exception: self.log.error("Exception in control handler:", exc_info=True) sys.stdout.flush() sys.stderr.flush() self._publish_status(u'idle') def dispatch_shell(self, stream, msg): """dispatch shell requests""" # flush control requests first if self.control_stream: self.control_stream.flush() idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.deserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return # Set the parent message for side effects. self.set_parent(idents, msg) self._publish_status(u'busy') header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) if msg_id in self.aborted: self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: # ensure default_int_handler during handler call sig = signal(SIGINT, default_int_handler) self.log.debug("%s: %s", msg_type, msg) try: handler(stream, idents, msg) except Exception: self.log.error("Exception in message handler:", exc_info=True) finally: signal(SIGINT, sig) sys.stdout.flush() sys.stderr.flush() self._publish_status(u'idle') def enter_eventloop(self): """enter eventloop""" self.log.info("entering eventloop %s", self.eventloop) for stream in self.shell_streams: # flush any pending replies, # which may be skipped by entering the eventloop stream.flush(zmq.POLLOUT) # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel self.log.error("KeyboardInterrupt caught in kernel") continue else: # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break self.log.info("exiting eventloop") def start(self): """register dispatchers for streams""" if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_shell(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) # publish idle status self._publish_status('starting') def do_one_iteration(self): """step eventloop just once""" if self.control_stream: self.control_stream.flush() for stream in self.shell_streams: # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _make_metadata(self, other=None): """init metadata dict, for execute/apply_reply""" new_md = { 'dependencies_met': True, 'engine': self.ident, 'started': datetime.now(), } if other: new_md.update(other) return new_md def _publish_execute_input(self, code, parent, execution_count): """Publish the code request on the iopub stream.""" self.session.send(self.iopub_socket, u'execute_input', { u'code': code, u'execution_count': execution_count }, parent=parent, ident=self._topic('execute_input')) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send( self.iopub_socket, u'status', {u'execution_state': status}, parent=parent or self._parent_header, ident=self._topic('status'), ) def set_parent(self, ident, parent): """Set the current parent_header Side effects (IOPub messages) and replies are associated with the request that caused them via the parent_header. The parent identity is used to route input_request messages on the stdin channel. """ self._parent_ident = ident self._parent_header = parent def send_response(self, stream, msg_or_type, content=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Send a response to the message we're currently processing. This accepts all the parameters of :meth:`ipython_kernel.session.Session.send` except ``parent``. This relies on :meth:`set_parent` having been called for the current message. """ return self.session.send(stream, msg_or_type, content, self._parent_header, ident, buffers, track, header, metadata) def execute_request(self, stream, ident, parent): """handle an execute_request""" try: content = parent[u'content'] code = py3compat.cast_unicode_py2(content[u'code']) silent = content[u'silent'] store_history = content.get(u'store_history', not silent) user_expressions = content.get('user_expressions', {}) allow_stdin = content.get('allow_stdin', False) except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return stop_on_error = content.get('stop_on_error', True) md = self._make_metadata(parent['metadata']) # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self.execution_count += 1 self._publish_execute_input(code, parent, self.execution_count) reply_content = self.do_execute(code, silent, store_history, user_expressions, allow_stdin) # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) md['status'] = reply_content['status'] if reply_content['status'] == 'error' and \ reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=md, ident=ident) self.log.debug("%s", reply_msg) if not silent and reply_msg['content'][ 'status'] == u'error' and stop_on_error: self._abort_queues() def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False): """Execute user code. Must be overridden by subclasses. """ raise NotImplementedError def complete_request(self, stream, ident, parent): content = parent['content'] code = content['code'] cursor_pos = content['cursor_pos'] matches = self.do_complete(code, cursor_pos) matches = json_clean(matches) completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) def do_complete(self, code, cursor_pos): """Override in subclasses to find completions. """ return { 'matches': [], 'cursor_end': cursor_pos, 'cursor_start': cursor_pos, 'metadata': {}, 'status': 'ok' } def inspect_request(self, stream, ident, parent): content = parent['content'] reply_content = self.do_inspect(content['code'], content['cursor_pos'], content.get('detail_level', 0)) # Before we send this object over, we scrub it for JSON usage reply_content = json_clean(reply_content) msg = self.session.send(stream, 'inspect_reply', reply_content, parent, ident) self.log.debug("%s", msg) def do_inspect(self, code, cursor_pos, detail_level=0): """Override in subclasses to allow introspection. """ return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False} def history_request(self, stream, ident, parent): content = parent['content'] reply_content = self.do_history(**content) reply_content = json_clean(reply_content) msg = self.session.send(stream, 'history_reply', reply_content, parent, ident) self.log.debug("%s", msg) def do_history(self, hist_access_type, output, raw, session=None, start=None, stop=None, n=None, pattern=None, unique=False): """Override in subclasses to access history. """ return {'history': []} def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) @property def kernel_info(self): return { 'protocol_version': release.kernel_protocol_version, 'implementation': self.implementation, 'implementation_version': self.implementation_version, 'language_info': self.language_info, 'banner': self.banner, 'help_links': self.help_links, } def kernel_info_request(self, stream, ident, parent): msg = self.session.send(stream, 'kernel_info_reply', self.kernel_info, parent, ident) self.log.debug("%s", msg) def shutdown_request(self, stream, ident, parent): content = self.do_shutdown(parent['content']['restart']) self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', content, parent) self._at_shutdown() # call sys.exit after a short delay loop = ioloop.IOLoop.instance() loop.add_timeout(time.time() + 0.1, loop.stop) def do_shutdown(self, restart): """Override in subclasses to do things when the frontend shuts down the kernel. """ return {'status': 'ok', 'restart': restart} def is_complete_request(self, stream, ident, parent): content = parent['content'] code = content['code'] reply_content = self.do_is_complete(code) reply_content = json_clean(reply_content) reply_msg = self.session.send(stream, 'is_complete_reply', reply_content, parent, ident) self.log.debug("%s", reply_msg) def do_is_complete(self, code): """Override in subclasses to find completions. """ return { 'status': 'unknown', } #--------------------------------------------------------------------------- # Engine methods #--------------------------------------------------------------------------- def apply_request(self, stream, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] except: self.log.error("Got bad msg: %s", parent, exc_info=True) return md = self._make_metadata(parent['metadata']) reply_content, result_buf = self.do_apply(content, bufs, msg_id, md) # put 'ok'/'error' status in header, for scheduler introspection: md['status'] = reply_content['status'] # flush i/o sys.stdout.flush() sys.stderr.flush() self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident, buffers=result_buf, metadata=md) def do_apply(self, content, bufs, msg_id, reply_metadata): """Override in subclasses to support the IPython parallel framework. """ raise NotImplementedError #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- def abort_request(self, stream, ident, parent): """abort a specific msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, string_types): msg_ids = [msg_ids] if not msg_ids: self._abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" content = self.do_clear() self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content=content) def do_clear(self): """Override in subclasses to clear the namespace This is only required for IPython.parallel. """ raise NotImplementedError #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _topic(self, topic): """prefixed topic for IOPub messages""" if self.int_id >= 0: base = "engine.%i" % self.int_id else: base = "kernel.%s" % self.ident return py3compat.cast_bytes("%s.%s" % (base, topic)) def _abort_queues(self): for stream in self.shell_streams: if stream: self._abort_queue(stream) def _abort_queue(self, stream): poller = zmq.Poller() poller.register(stream.socket, zmq.POLLIN) while True: idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. poller.poll(50) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this " "frontend does not support stdin.") def getpass(self, prompt=''): """Forward getpass to frontends Raises ------ StdinNotImplentedError if active frontend doesn't support stdin. """ if not self._allow_stdin: raise StdinNotImplementedError( "getpass was called, but this frontend does not support input requests." ) return self._input_request( prompt, self._parent_ident, self._parent_header, password=True, ) def raw_input(self, prompt=''): """Forward raw_input to frontends Raises ------ StdinNotImplentedError if active frontend doesn't support stdin. """ if not self._allow_stdin: raise StdinNotImplementedError( "raw_input was called, but this frontend does not support input requests." ) return self._input_request( prompt, self._parent_ident, self._parent_header, password=False, ) def _input_request(self, prompt, ident, parent, password=False): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # flush the stdin socket, to purge stale replies while True: try: self.stdin_socket.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: break else: raise # Send the input request. content = json_clean(dict(prompt=prompt, password=password)) self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) except KeyboardInterrupt: # re-raise KeyboardInterrupt, to truncate traceback raise KeyboardInterrupt else: break try: value = py3compat.unicode_to_str(reply['content']['value']) except: self.log.error("Bad input_reply: %s", parent) value = '' if value == '\x04': # EOF raise EOFError return value def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [s.flush(zmq.POLLOUT) for s in self.shell_streams]
class FileContentsManager(ContentsManager): root_dir = Unicode(config=True) def _root_dir_default(self): try: return self.parent.notebook_dir except AttributeError: return getcwd() @contextmanager def perm_to_403(self, os_path=''): """context manager for turning permission errors into 403""" try: yield except OSError as e: if e.errno in {errno.EPERM, errno.EACCES}: # make 403 error message without root prefix # this may not work perfectly on unicode paths on Python 2, # but nobody should be doing that anyway. if not os_path: os_path = str_to_unicode(e.filename or 'unknown file') path = to_api_path(os_path, self.root_dir) raise web.HTTPError(403, u'Permission denied: %s' % path) else: raise @contextmanager def open(self, os_path, *args, **kwargs): """wrapper around io.open that turns permission errors into 403""" with self.perm_to_403(os_path): with io.open(os_path, *args, **kwargs) as f: yield f @contextmanager def atomic_writing(self, os_path, *args, **kwargs): """wrapper around atomic_writing that turns permission errors into 403""" with self.perm_to_403(os_path): with atomic_writing(os_path, *args, **kwargs) as f: yield f save_script = Bool(False, config=True, help='DEPRECATED, IGNORED') def _save_script_changed(self): self.log.warn(""" Automatically saving notebooks as scripts has been removed. Use `ipython nbconvert --to python [notebook]` instead. """) def _root_dir_changed(self, name, old, new): """Do a bit of validation of the root_dir.""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. self.root_dir = os.path.abspath(new) return if not os.path.isdir(new): raise TraitError("%r is not a directory" % new) checkpoint_dir = Unicode( '.ipynb_checkpoints', config=True, help="""The directory name in which to keep file checkpoints This is a path relative to the file's own directory. By default, it is .ipynb_checkpoints """) def _copy(self, src, dest): """copy src to dest like shutil.copy2, but log errors in copystat """ shutil.copyfile(src, dest) try: shutil.copystat(src, dest) except OSError as e: self.log.debug("copystat on %s failed", dest, exc_info=True) def _get_os_path(self, path): """Given an API path, return its file system path. Parameters ---------- path : string The relative API path to the named file. Returns ------- path : string Native, absolute OS path to for a file. """ return to_os_path(path, self.root_dir) def dir_exists(self, path): """Does the API-style path refer to an extant directory? API-style wrapper for os.path.isdir Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to root_dir). Returns ------- exists : bool Whether the path is indeed a directory. """ path = path.strip('/') os_path = self._get_os_path(path=path) return os.path.isdir(os_path) def is_hidden(self, path): """Does the API style path correspond to a hidden directory or file? Parameters ---------- path : string The path to check. This is an API path (`/` separated, relative to root_dir). Returns ------- hidden : bool Whether the path exists and is hidden. """ path = path.strip('/') os_path = self._get_os_path(path=path) return is_hidden(os_path, self.root_dir) def file_exists(self, path): """Returns True if the file exists, else returns False. API-style wrapper for os.path.isfile Parameters ---------- path : string The relative path to the file (with '/' as separator) Returns ------- exists : bool Whether the file exists. """ path = path.strip('/') os_path = self._get_os_path(path) return os.path.isfile(os_path) def exists(self, path): """Returns True if the path exists, else returns False. API-style wrapper for os.path.exists Parameters ---------- path : string The API path to the file (with '/' as separator) Returns ------- exists : bool Whether the target exists. """ path = path.strip('/') os_path = self._get_os_path(path=path) return os.path.exists(os_path) def _base_model(self, path): """Build the common base of a contents model""" os_path = self._get_os_path(path) info = os.stat(os_path) last_modified = tz.utcfromtimestamp(info.st_mtime) created = tz.utcfromtimestamp(info.st_ctime) # Create the base model. model = {} model['name'] = path.rsplit('/', 1)[-1] model['path'] = path model['last_modified'] = last_modified model['created'] = created model['content'] = None model['format'] = None try: model['writable'] = os.access(os_path, os.W_OK) except OSError: self.log.error("Failed to check write permissions on %s", os_path) model['writable'] = False return model def _dir_model(self, path, content=True): """Build a model for a directory if content is requested, will include a listing of the directory """ os_path = self._get_os_path(path) four_o_four = u'directory does not exist: %r' % path if not os.path.isdir(os_path): raise web.HTTPError(404, four_o_four) elif is_hidden(os_path, self.root_dir): self.log.info( "Refusing to serve hidden directory %r, via 404 Error", os_path) raise web.HTTPError(404, four_o_four) model = self._base_model(path) model['type'] = 'directory' if content: model['content'] = contents = [] os_dir = self._get_os_path(path) for name in os.listdir(os_dir): os_path = os.path.join(os_dir, name) # skip over broken symlinks in listing if not os.path.exists(os_path): self.log.warn("%s doesn't exist", os_path) continue elif not os.path.isfile(os_path) and not os.path.isdir( os_path): self.log.debug("%s not a regular file", os_path) continue if self.should_list(name) and not is_hidden( os_path, self.root_dir): contents.append( self.get(path='%s/%s' % (path, name), content=False)) model['format'] = 'json' return model def _file_model(self, path, content=True, format=None): """Build a model for a file if content is requested, include the file contents. format: If 'text', the contents will be decoded as UTF-8. If 'base64', the raw bytes contents will be encoded as base64. If not specified, try to decode as UTF-8, and fall back to base64 """ model = self._base_model(path) model['type'] = 'file' if content: os_path = self._get_os_path(path) if not os.path.isfile(os_path): # could be FIFO raise web.HTTPError( 400, "Cannot get content of non-file %s" % os_path) with self.open(os_path, 'rb') as f: bcontent = f.read() if format != 'base64': try: model['content'] = bcontent.decode('utf8') except UnicodeError as e: if format == 'text': raise web.HTTPError(400, "%s is not UTF-8 encoded" % path) else: model['format'] = 'text' if model['content'] is None: model['content'] = base64.encodestring(bcontent).decode( 'ascii') model['format'] = 'base64' return model def _notebook_model(self, path, content=True): """Build a notebook model if content is requested, the notebook content will be populated as a JSON structure (not double-serialized) """ model = self._base_model(path) model['type'] = 'notebook' if content: os_path = self._get_os_path(path) with self.open(os_path, 'r', encoding='utf-8') as f: try: nb = nbformat.read(f, as_version=4) except Exception as e: raise web.HTTPError( 400, u"Unreadable Notebook: %s %r" % (os_path, e)) self.mark_trusted_cells(nb, path) model['content'] = nb model['format'] = 'json' self.validate_notebook_model(model) return model def get(self, path, content=True, type_=None, format=None): """ Takes a path for an entity and returns its model Parameters ---------- path : str the API path that describes the relative path for the target content : bool Whether to include the contents in the reply type_ : str, optional The requested type - 'file', 'notebook', or 'directory'. Will raise HTTPError 400 if the content doesn't match. format : str, optional The requested format for file contents. 'text' or 'base64'. Ignored if this returns a notebook or directory model. Returns ------- model : dict the contents model. If content=True, returns the contents of the file or directory as well. """ path = path.strip('/') if not self.exists(path): raise web.HTTPError(404, u'No such file or directory: %s' % path) os_path = self._get_os_path(path) if os.path.isdir(os_path): if type_ not in (None, 'directory'): raise web.HTTPError( 400, u'%s is a directory, not a %s' % (path, type_)) model = self._dir_model(path, content=content) elif type_ == 'notebook' or (type_ is None and path.endswith('.ipynb')): model = self._notebook_model(path, content=content) else: if type_ == 'directory': raise web.HTTPError(400, u'%s is not a directory') model = self._file_model(path, content=content, format=format) return model def _save_notebook(self, os_path, model, path=''): """save a notebook file""" # Save the notebook file nb = nbformat.from_dict(model['content']) self.check_and_sign(nb, path) with self.atomic_writing(os_path, encoding='utf-8') as f: nbformat.write(nb, f, version=nbformat.NO_CONVERT) def _save_file(self, os_path, model, path=''): """save a non-notebook file""" fmt = model.get('format', None) if fmt not in {'text', 'base64'}: raise web.HTTPError( 400, "Must specify format of file contents as 'text' or 'base64'") try: content = model['content'] if fmt == 'text': bcontent = content.encode('utf8') else: b64_bytes = content.encode('ascii') bcontent = base64.decodestring(b64_bytes) except Exception as e: raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e)) with self.atomic_writing(os_path, text=False) as f: f.write(bcontent) def _save_directory(self, os_path, model, path=''): """create a directory""" if is_hidden(os_path, self.root_dir): raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path) if not os.path.exists(os_path): with self.perm_to_403(): os.mkdir(os_path) elif not os.path.isdir(os_path): raise web.HTTPError(400, u'Not a directory: %s' % (os_path)) else: self.log.debug("Directory %r already exists", os_path) def save(self, model, path=''): """Save the file model and return the model with no content.""" path = path.strip('/') if 'type' not in model: raise web.HTTPError(400, u'No file type provided') if 'content' not in model and model['type'] != 'directory': raise web.HTTPError(400, u'No file content provided') # One checkpoint should always exist if self.file_exists(path) and not self.list_checkpoints(path): self.create_checkpoint(path) os_path = self._get_os_path(path) self.log.debug("Saving %s", os_path) try: if model['type'] == 'notebook': self._save_notebook(os_path, model, path) elif model['type'] == 'file': self._save_file(os_path, model, path) elif model['type'] == 'directory': self._save_directory(os_path, model, path) else: raise web.HTTPError( 400, "Unhandled contents type: %s" % model['type']) except web.HTTPError: raise except Exception as e: self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True) raise web.HTTPError( 500, u'Unexpected error while saving file: %s %s' % (path, e)) validation_message = None if model['type'] == 'notebook': self.validate_notebook_model(model) validation_message = model.get('message', None) model = self.get(path, content=False) if validation_message: model['message'] = validation_message return model def update(self, model, path): """Update the file's path For use in PATCH requests, to enable renaming a file without re-uploading its contents. Only used for renaming at the moment. """ path = path.strip('/') new_path = model.get('path', path).strip('/') if path != new_path: self.rename(path, new_path) model = self.get(new_path, content=False) return model def delete(self, path): """Delete file at path.""" path = path.strip('/') os_path = self._get_os_path(path) rm = os.unlink if os.path.isdir(os_path): listing = os.listdir(os_path) # don't delete non-empty directories (checkpoints dir doesn't count) if listing and listing != [self.checkpoint_dir]: raise web.HTTPError(400, u'Directory %s not empty' % os_path) elif not os.path.isfile(os_path): raise web.HTTPError(404, u'File does not exist: %s' % os_path) # clear checkpoints for checkpoint in self.list_checkpoints(path): checkpoint_id = checkpoint['id'] cp_path = self.get_checkpoint_path(checkpoint_id, path) if os.path.isfile(cp_path): self.log.debug("Unlinking checkpoint %s", cp_path) with self.perm_to_403(): rm(cp_path) if os.path.isdir(os_path): self.log.debug("Removing directory %s", os_path) with self.perm_to_403(): shutil.rmtree(os_path) else: self.log.debug("Unlinking file %s", os_path) with self.perm_to_403(): rm(os_path) def rename(self, old_path, new_path): """Rename a file.""" old_path = old_path.strip('/') new_path = new_path.strip('/') if new_path == old_path: return new_os_path = self._get_os_path(new_path) old_os_path = self._get_os_path(old_path) # Should we proceed with the move? if os.path.exists(new_os_path): raise web.HTTPError(409, u'File already exists: %s' % new_path) # Move the file try: with self.perm_to_403(): shutil.move(old_os_path, new_os_path) except web.HTTPError: raise except Exception as e: raise web.HTTPError( 500, u'Unknown error renaming file: %s %s' % (old_path, e)) # Move the checkpoints old_checkpoints = self.list_checkpoints(old_path) for cp in old_checkpoints: checkpoint_id = cp['id'] old_cp_path = self.get_checkpoint_path(checkpoint_id, old_path) new_cp_path = self.get_checkpoint_path(checkpoint_id, new_path) if os.path.isfile(old_cp_path): self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path) with self.perm_to_403(): shutil.move(old_cp_path, new_cp_path) # Checkpoint-related utilities def get_checkpoint_path(self, checkpoint_id, path): """find the path to a checkpoint""" path = path.strip('/') parent, name = ('/' + path).rsplit('/', 1) parent = parent.strip('/') basename, ext = os.path.splitext(name) filename = u"{name}-{checkpoint_id}{ext}".format( name=basename, checkpoint_id=checkpoint_id, ext=ext, ) os_path = self._get_os_path(path=parent) cp_dir = os.path.join(os_path, self.checkpoint_dir) with self.perm_to_403(): ensure_dir_exists(cp_dir) cp_path = os.path.join(cp_dir, filename) return cp_path def get_checkpoint_model(self, checkpoint_id, path): """construct the info dict for a given checkpoint""" path = path.strip('/') cp_path = self.get_checkpoint_path(checkpoint_id, path) stats = os.stat(cp_path) last_modified = tz.utcfromtimestamp(stats.st_mtime) info = dict( id=checkpoint_id, last_modified=last_modified, ) return info # public checkpoint API def create_checkpoint(self, path): """Create a checkpoint from the current state of a file""" path = path.strip('/') if not self.file_exists(path): raise web.HTTPError(404) src_path = self._get_os_path(path) # only the one checkpoint ID: checkpoint_id = u"checkpoint" cp_path = self.get_checkpoint_path(checkpoint_id, path) self.log.debug("creating checkpoint for %s", path) with self.perm_to_403(): self._copy(src_path, cp_path) # return the checkpoint info return self.get_checkpoint_model(checkpoint_id, path) def list_checkpoints(self, path): """list the checkpoints for a given file This contents manager currently only supports one checkpoint per file. """ path = path.strip('/') checkpoint_id = "checkpoint" os_path = self.get_checkpoint_path(checkpoint_id, path) if not os.path.exists(os_path): return [] else: return [self.get_checkpoint_model(checkpoint_id, path)] def restore_checkpoint(self, checkpoint_id, path): """restore a file to a checkpointed state""" path = path.strip('/') self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) nb_path = self._get_os_path(path) cp_path = self.get_checkpoint_path(checkpoint_id, path) if not os.path.isfile(cp_path): self.log.debug("checkpoint file does not exist: %s", cp_path) raise web.HTTPError( 404, u'checkpoint does not exist: %s@%s' % (path, checkpoint_id)) # ensure notebook is readable (never restore from an unreadable notebook) if cp_path.endswith('.ipynb'): with self.open(cp_path, 'r', encoding='utf-8') as f: nbformat.read(f, as_version=4) self.log.debug("copying %s -> %s", cp_path, nb_path) with self.perm_to_403(): self._copy(cp_path, nb_path) def delete_checkpoint(self, checkpoint_id, path): """delete a file's checkpoint""" path = path.strip('/') cp_path = self.get_checkpoint_path(checkpoint_id, path) if not os.path.isfile(cp_path): raise web.HTTPError( 404, u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id)) self.log.debug("unlinking %s", cp_path) os.unlink(cp_path) def info_string(self): return "Serving notebooks from local directory: %s" % self.root_dir def get_kernel_path(self, path, model=None): """Return the initial working dir a kernel associated with a given notebook""" if '/' in path: parent_dir = path.rsplit('/', 1)[0] else: parent_dir = '' return self._get_os_path(parent_dir)
class Kernel(SessionFactory): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # kwargs: exec_lines = List(Unicode, config=True, help="List of lines to execute") # identities: int_id = Integer(-1) bident = CBytes() ident = Unicode() def _ident_changed(self, name, old, new): self.bident = asbytes(new) user_ns = Dict(config=True, help="""Set the user's namespace of the Kernel""") control_stream = Instance(zmqstream.ZMQStream) task_stream = Instance(zmqstream.ZMQStream) iopub_stream = Instance(zmqstream.ZMQStream) client = Instance('IPython.parallel.Client') # internals shell_streams = List() compiler = Instance(CommandCompiler, (), {}) completer = Instance(KernelCompleter) aborted = Set() shell_handlers = Dict() control_handlers = Dict() def _set_prefix(self): self.prefix = "engine.%s" % self.int_id def _connect_completer(self): self.completer = KernelCompleter(self.user_ns) def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) self._set_prefix() self._connect_completer() self.on_trait_change(self._set_prefix, 'id') self.on_trait_change(self._connect_completer, 'user_ns') # Build dict of handlers for message types for msg_type in [ 'execute_request', 'complete_request', 'apply_request', 'clear_request' ]: self.shell_handlers[msg_type] = getattr(self, msg_type) for msg_type in ['shutdown_request', 'abort_request' ] + self.shell_handlers.keys(): self.control_handlers[msg_type] = getattr(self, msg_type) self._initial_exec_lines() def _wrap_exception(self, method=None): e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method) content = wrap_exception(e_info) return content def _initial_exec_lines(self): s = _Passer() content = dict(silent=True, user_variable=[], user_expressions=[]) for line in self.exec_lines: self.log.debug("executing initialization: %s" % line) content.update({'code': line}) msg = self.session.msg('execute_request', content) self.execute_request(s, [], msg) #-------------------- control handlers ----------------------------- def abort_queues(self): for stream in self.shell_streams: if stream: self.abort_queue(stream) def abort_queue(self, stream): while True: idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info(str(msg)) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) # self.reply_socket.send(ident,zmq.SNDMORE) # self.reply_socket.send_json(reply_msg) reply_msg = self.session.send(stream, reply_type, content={'status': 'aborted'}, parent=msg, ident=idents) self.log.debug(str(reply_msg)) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. time.sleep(0.05) def abort_request(self, stream, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, basestring): msg_ids = [msg_ids] if not msg_ids: self.abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug(str(reply_msg)) def shutdown_request(self, stream, ident, parent): """kill ourself. This should really be handled in an external process""" try: self.abort_queues() except: content = self._wrap_exception('shutdown') else: content = dict(parent['content']) content['status'] = 'ok' msg = self.session.send(stream, 'shutdown_reply', content=content, parent=parent, ident=ident) self.log.debug(str(msg)) dc = ioloop.DelayedCallback(lambda: sys.exit(0), 1000, self.loop) dc.start() def dispatch_control(self, msg): idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return else: self.log.debug("Control received, %s", msg) header = msg['header'] msg_id = header['msg_id'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r" % msg_type) else: handler(self.control_stream, idents, msg) #-------------------- queue helpers ------------------------------ def check_dependencies(self, dependencies): if not dependencies: return True if len(dependencies) == 2 and dependencies[0] in 'any all'.split(): anyorall = dependencies[0] dependencies = dependencies[1] else: anyorall = 'all' results = self.client.get_results(dependencies, status_only=True) if results['status'] != 'ok': return False if anyorall == 'any': if not results['completed']: return False else: if results['pending']: return False return True def check_aborted(self, msg_id): return msg_id in self.aborted #-------------------- queue handlers ----------------------------- def clear_request(self, stream, idents, parent): """Clear our namespace.""" self.user_ns = {} msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content=dict(status='ok')) self._initial_exec_lines() def execute_request(self, stream, ident, parent): self.log.debug('execute request %s' % parent) try: code = parent[u'content'][u'code'] except: self.log.error("Got bad msg: %s" % parent, exc_info=True) return self.session.send(self.iopub_stream, u'pyin', {u'code': code}, parent=parent, ident=asbytes('%s.pyin' % self.prefix)) started = datetime.now() try: comp_code = self.compiler(code, '<zmq-kernel>') # allow for not overriding displayhook if hasattr(sys.displayhook, 'set_parent'): sys.displayhook.set_parent(parent) sys.stdout.set_parent(parent) sys.stderr.set_parent(parent) exec comp_code in self.user_ns, self.user_ns except: exc_content = self._wrap_exception('execute') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, ident=asbytes('%s.pyerr' % self.prefix)) reply_content = exc_content else: reply_content = {'status': 'ok'} reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent=parent, ident=ident, subheader=dict(started=started)) self.log.debug(str(reply_msg)) if reply_msg['content']['status'] == u'error': self.abort_queues() def complete_request(self, stream, ident, parent): matches = {'matches': self.complete(parent), 'status': 'ok'} completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) # print >> sys.__stdout__, completion_msg def complete(self, msg): return self.completer.complete(msg.content.line, msg.content.text) def apply_request(self, stream, ident, parent): # flush previous reply, so this request won't block it stream.flush(zmq.POLLOUT) try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] # bound = parent['header'].get('bound', False) except: self.log.error("Got bad msg: %s" % parent, exc_info=True) return # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_stream.send(pyin_msg) # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent) sub = { 'dependencies_met': True, 'engine': self.ident, 'started': datetime.now() } try: # allow for not overriding displayhook if hasattr(sys.displayhook, 'set_parent'): sys.displayhook.set_parent(parent) sys.stdout.set_parent(parent) sys.stderr.set_parent(parent) # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns working = self.user_ns # suffix = prefix = "_" + str(msg_id).replace("-", "") + "_" f, args, kwargs = unpack_apply_message(bufs, working, copy=False) # if bound: # bound_ns = Namespace(working) # args = [bound_ns]+list(args) fname = getattr(f, '__name__', 'f') fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" ns = {fname: f, argname: args, kwargname: kwargs, resultname: None} # print ns working.update(ns) code = "%s=%s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec code in working, working result = working.get(resultname) finally: for key in ns.iterkeys(): working.pop(key) # if bound: # working.update(bound_ns) packed_result, buf = serialize_object(result) result_buf = [packed_result] + buf except: exc_content = self._wrap_exception('apply') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, ident=asbytes('%s.pyerr' % self.prefix)) reply_content = exc_content result_buf = [] if exc_content['ename'] == 'UnmetDependency': sub['dependencies_met'] = False else: reply_content = {'status': 'ok'} # put 'ok'/'error' status in header, for scheduler introspection: sub['status'] = reply_content['status'] reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident, buffers=result_buf, subheader=sub) # flush i/o # should this be before reply_msg is sent, like in the single-kernel code, # or should nothing get in the way of real results? sys.stdout.flush() sys.stderr.flush() def dispatch_queue(self, stream, msg): self.control_stream.flush() idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return else: self.log.debug("Message received, %s", msg) header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] if self.check_aborted(msg_id): self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} reply_msg = self.session.send(stream, reply_type, subheader=status, content=status, parent=msg, ident=idents) return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r" % msg_type) else: handler(stream, idents, msg) def start(self): #### stream mode: if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) self.control_stream.on_err(printer) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_queue(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) s.on_err(printer) if self.iopub_stream: self.iopub_stream.on_err(printer)
class HistoryAccessor(HistoryAccessorBase): """Access the history database without adding to it. This is intended for use by standalone history tools. IPython shells use HistoryManager, below, which is a subclass of this.""" # String holding the path to the history file hist_file = Unicode( config=True, help="""Path to file to use for SQLite history database. By default, IPython will put the history database in the IPython profile directory. If you would rather share one history among profiles, you can set this value in each, so that they are consistent. Due to an issue with fcntl, SQLite is known to misbehave on some NFS mounts. If you see IPython hanging, try setting this to something on a local disk, e.g:: ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite """) enabled = Bool(True, config=True, help="""enable the SQLite history set enabled=False to disable the SQLite history, in which case there will be no stored history, no SQLite connection, and no background saving thread. This may be necessary in some threaded environments where IPython is embedded. """) connection_options = Dict( config=True, help="""Options for configuring the SQLite connection These options are passed as keyword args to sqlite3.connect when establishing database conenctions. """) # The SQLite database db = Any() def _db_changed(self, name, old, new): """validate the db, since it can be an Instance of two different types""" connection_types = (DummyDB, ) if sqlite3 is not None: connection_types = (DummyDB, sqlite3.Connection) if not isinstance(new, connection_types): msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \ (self.__class__.__name__, new) raise TraitError(msg) def __init__(self, profile='default', hist_file=u'', **traits): """Create a new history accessor. Parameters ---------- profile : str The name of the profile from which to open history. hist_file : str Path to an SQLite history database stored by IPython. If specified, hist_file overrides profile. config : :class:`~IPython.config.loader.Config` Config object. hist_file can also be set through this. """ # We need a pointer back to the shell for various tasks. super(HistoryAccessor, self).__init__(**traits) # defer setting hist_file from kwarg until after init, # otherwise the default kwarg value would clobber any value # set by config if hist_file: self.hist_file = hist_file if self.hist_file == u'': # No one has set the hist_file, yet. self.hist_file = self._get_hist_file_name(profile) if sqlite3 is None and self.enabled: warn( "IPython History requires SQLite, your history will not be saved" ) self.enabled = False self.init_db() def _get_hist_file_name(self, profile='default'): """Find the history file for the given profile name. This is overridden by the HistoryManager subclass, to use the shell's active profile. Parameters ---------- profile : str The name of a profile which has a history file. """ return os.path.join(locate_profile(profile), 'history.sqlite') @catch_corrupt_db def init_db(self): """Connect to the database, and create tables if necessary.""" if not self.enabled: self.db = DummyDB() return # use detect_types so that timestamps return datetime objects kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) kwargs.update(self.connection_options) self.db = sqlite3.connect(self.hist_file, **kwargs) self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer primary key autoincrement, start timestamp, end timestamp, num_cmds integer, remark text)""") self.db.execute("""CREATE TABLE IF NOT EXISTS history (session integer, line integer, source text, source_raw text, PRIMARY KEY (session, line))""") # Output history is optional, but ensure the table's there so it can be # enabled later. self.db.execute("""CREATE TABLE IF NOT EXISTS output_history (session integer, line integer, output text, PRIMARY KEY (session, line))""") self.db.commit() def writeout_cache(self): """Overridden by HistoryManager to dump the cache before certain database lookups.""" pass ## ------------------------------- ## Methods for retrieving history: ## ------------------------------- def _run_sql(self, sql, params, raw=True, output=False): """Prepares and runs an SQL query for the history database. Parameters ---------- sql : str Any filtering expressions to go after SELECT ... FROM ... params : tuple Parameters passed to the SQL query (to replace "?") raw, output : bool See :meth:`get_range` Returns ------- Tuples as :meth:`get_range` """ toget = 'source_raw' if raw else 'source' sqlfrom = "history" if output: sqlfrom = "history LEFT JOIN output_history USING (session, line)" toget = "history.%s, output_history.output" % toget cur = self.db.execute("SELECT session, line, %s FROM %s " %\ (toget, sqlfrom) + sql, params) if output: # Regroup into 3-tuples, and parse JSON return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur) return cur @needs_sqlite @catch_corrupt_db def get_session_info(self, session): """Get info about a session. Parameters ---------- session : int Session number to retrieve. Returns ------- session_id : int Session ID number start : datetime Timestamp for the start of the session. end : datetime Timestamp for the end of the session, or None if IPython crashed. num_cmds : int Number of commands run, or None if IPython crashed. remark : unicode A manually set description. """ query = "SELECT * from sessions where session == ?" return self.db.execute(query, (session, )).fetchone() @catch_corrupt_db def get_last_session_id(self): """Get the last session ID currently in the database. Within IPython, this should be the same as the value stored in :attr:`HistoryManager.session_number`. """ for record in self.get_tail(n=1, include_latest=True): return record[0] @catch_corrupt_db def get_tail(self, n=10, raw=True, output=False, include_latest=False): """Get the last n lines from the history database. Parameters ---------- n : int The number of lines to get raw, output : bool See :meth:`get_range` include_latest : bool If False (default), n+1 lines are fetched, and the latest one is discarded. This is intended to be used where the function is called by a user command, which it should not return. Returns ------- Tuples as :meth:`get_range` """ self.writeout_cache() if not include_latest: n += 1 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?", (n, ), raw=raw, output=output) if not include_latest: return reversed(list(cur)[1:]) return reversed(list(cur)) @catch_corrupt_db def search(self, pattern="*", raw=True, search_raw=True, output=False, n=None, unique=False): """Search the database using unix glob-style matching (wildcards * and ?). Parameters ---------- pattern : str The wildcarded pattern to match when searching search_raw : bool If True, search the raw input, otherwise, the parsed input raw, output : bool See :meth:`get_range` n : None or int If an integer is given, it defines the limit of returned entries. unique : bool When it is true, return only unique entries. Returns ------- Tuples as :meth:`get_range` """ tosearch = "source_raw" if search_raw else "source" if output: tosearch = "history." + tosearch self.writeout_cache() sqlform = "WHERE %s GLOB ?" % tosearch params = (pattern, ) if unique: sqlform += ' GROUP BY {0}'.format(tosearch) if n is not None: sqlform += " ORDER BY session DESC, line DESC LIMIT ?" params += (n, ) elif unique: sqlform += " ORDER BY session, line" cur = self._run_sql(sqlform, params, raw=raw, output=output) if n is not None: return reversed(list(cur)) return cur @catch_corrupt_db def get_range(self, session, start=1, stop=None, raw=True, output=False): """Retrieve input by session. Parameters ---------- session : int Session number to retrieve. start : int First line to retrieve. stop : int End of line range (excluded from output itself). If None, retrieve to the end of the session. raw : bool If True, return untranslated input output : bool If True, attempt to include output. This will be 'real' Python objects for the current session, or text reprs from previous sessions if db_log_output was enabled at the time. Where no output is found, None is used. Returns ------- entries An iterator over the desired lines. Each line is a 3-tuple, either (session, line, input) if output is False, or (session, line, (input, output)) if output is True. """ if stop: lineclause = "line >= ? AND line < ?" params = (session, start, stop) else: lineclause = "line>=?" params = (session, start) return self._run_sql("WHERE session==? AND %s" % lineclause, params, raw=raw, output=output) def get_range_by_str(self, rangestr, raw=True, output=False): """Get lines of history from a string of ranges, as used by magic commands %hist, %save, %macro, etc. Parameters ---------- rangestr : str A string specifying ranges, e.g. "5 ~2/1-4". See :func:`magic_history` for full details. raw, output : bool As :meth:`get_range` Returns ------- Tuples as :meth:`get_range` """ for sess, s, e in extract_hist_ranges(rangestr): for line in self.get_range(sess, s, e, raw=raw, output=output): yield line
class WinHPCJob(Configurable): job_id = Unicode('') job_name = Unicode('MyJob', config=True) min_cores = Int(1, config=True) max_cores = Int(1, config=True) min_sockets = Int(1, config=True) max_sockets = Int(1, config=True) min_nodes = Int(1, config=True) max_nodes = Int(1, config=True) unit_type = Unicode("Core", config=True) auto_calculate_min = Bool(True, config=True) auto_calculate_max = Bool(True, config=True) run_until_canceled = Bool(False, config=True) is_exclusive = Bool(False, config=True) username = Unicode(find_username(), config=True) job_type = Unicode('Batch', config=True) priority = Enum( ('Lowest', 'BelowNormal', 'Normal', 'AboveNormal', 'Highest'), default_value='Highest', config=True) requested_nodes = Unicode('', config=True) project = Unicode('IPython', config=True) xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/') version = Unicode("2.000") tasks = List([]) @property def owner(self): return self.username def _write_attr(self, root, attr, key): s = as_str(getattr(self, attr, '')) if s: root.set(key, s) def as_element(self): # We have to add _A_ type things to get the right order than # the MSFT XML parser expects. root = ET.Element('Job') self._write_attr(root, 'version', '_A_Version') self._write_attr(root, 'job_name', '_B_Name') self._write_attr(root, 'unit_type', '_C_UnitType') self._write_attr(root, 'min_cores', '_D_MinCores') self._write_attr(root, 'max_cores', '_E_MaxCores') self._write_attr(root, 'min_sockets', '_F_MinSockets') self._write_attr(root, 'max_sockets', '_G_MaxSockets') self._write_attr(root, 'min_nodes', '_H_MinNodes') self._write_attr(root, 'max_nodes', '_I_MaxNodes') self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled') self._write_attr(root, 'is_exclusive', '_K_IsExclusive') self._write_attr(root, 'username', '_L_UserName') self._write_attr(root, 'job_type', '_M_JobType') self._write_attr(root, 'priority', '_N_Priority') self._write_attr(root, 'requested_nodes', '_O_RequestedNodes') self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax') self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin') self._write_attr(root, 'project', '_R_Project') self._write_attr(root, 'owner', '_S_Owner') self._write_attr(root, 'xmlns', '_T_xmlns') dependencies = ET.SubElement(root, "Dependencies") etasks = ET.SubElement(root, "Tasks") for t in self.tasks: etasks.append(t.as_element()) return root def tostring(self): """Return the string representation of the job description XML.""" root = self.as_element() indent(root) txt = ET.tostring(root, encoding="utf-8") # Now remove the tokens used to order the attributes. txt = re.sub(r'_[A-Z]_', '', txt) txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt return txt def write(self, filename): """Write the XML job description to a file.""" txt = self.tostring() with open(filename, 'w') as f: f.write(txt) def add_task(self, task): """Add a task to the job. Parameters ---------- task : :class:`WinHPCTask` The task object to add. """ self.tasks.append(task)
class TerminalInteractiveShell(InteractiveShell): autoedit_syntax = CBool(False, config=True, help="auto editing of files with syntax errors.") banner = Unicode('') banner1 = Unicode( default_banner, config=True, help="""The part of the banner to be printed before the profile""") banner2 = Unicode( '', config=True, help="""The part of the banner to be printed after the profile""") confirm_exit = CBool( True, config=True, help=""" Set to confirm when you try to exit IPython with an EOF (Control-D in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', you can force a direct exit without any confirmation.""", ) # This display_banner only controls whether or not self.show_banner() # is called when mainloop/interact are called. The default is False # because for the terminal based application, the banner behavior # is controlled by Global.display_banner, which IPythonApp looks at # to determine if *it* should call show_banner() by hand or not. display_banner = CBool(False) # This isn't configurable! embedded = CBool(False) embedded_active = CBool(False) editor = Unicode( get_default_editor(), config=True, help="Set the editor used by IPython (default to $EDITOR/vi/notepad).") pager = Unicode('less', config=True, help="The shell program to be used for paging.") screen_length = Integer( 0, config=True, help="""Number of lines of your screen, used to control printing of very long strings. Strings longer than this number of lines will be sent through a pager instead of directly printed. The default value for this is 0, which means IPython will auto-detect your screen size every time it needs to print certain potentially long strings (this doesn't change the behavior of the 'print' keyword, it's only triggered internally). If for some reason this isn't working well (it needs curses support), specify it yourself. Otherwise don't change the default.""", ) term_title = CBool(False, config=True, help="Enable auto setting the terminal title.") # This `using_paste_magics` is used to detect whether the code is being # executed via paste magics functions using_paste_magics = CBool(False) # In the terminal, GUI control is done via PyOS_InputHook @staticmethod def enable_gui(gui=None, app=None): """Switch amongst GUI input hooks by name. """ # Deferred import from IPython.lib.inputhook import enable_gui as real_enable_gui return real_enable_gui(gui, app) def __init__(self, config=None, ipython_dir=None, profile_dir=None, user_ns=None, user_module=None, custom_exceptions=((), None), usage=None, banner1=None, banner2=None, display_banner=None, **kwargs): super(TerminalInteractiveShell, self).__init__(config=config, ipython_dir=ipython_dir, profile_dir=profile_dir, user_ns=user_ns, user_module=user_module, custom_exceptions=custom_exceptions, **kwargs) # use os.system instead of utils.process.system by default, # because piped system doesn't make sense in the Terminal: self.system = self.system_raw self.init_term_title() self.init_usage(usage) self.init_banner(banner1, banner2, display_banner) #------------------------------------------------------------------------- # Overrides of init stages #------------------------------------------------------------------------- def init_display_formatter(self): super(TerminalInteractiveShell, self).init_display_formatter() # terminal only supports plaintext self.display_formatter.active_types = ['text/plain'] #------------------------------------------------------------------------- # Things related to the terminal #------------------------------------------------------------------------- @property def usable_screen_length(self): if self.screen_length == 0: return 0 else: num_lines_bot = self.separate_in.count('\n') + 1 return self.screen_length - num_lines_bot def init_term_title(self): # Enable or disable the terminal title. if self.term_title: toggle_set_term_title(True) set_term_title('IPython: ' + abbrev_cwd()) else: toggle_set_term_title(False) #------------------------------------------------------------------------- # Things related to aliases #------------------------------------------------------------------------- def init_alias(self): # The parent class defines aliases that can be safely used with any # frontend. super(TerminalInteractiveShell, self).init_alias() # Now define aliases that only make sense on the terminal, because they # need direct access to the console in a way that we can't emulate in # GUI or web frontend if os.name == 'posix': aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'), ('man', 'man')] elif os.name == 'nt': aliases = [('cls', 'cls')] for name, cmd in aliases: self.alias_manager.define_alias(name, cmd) #------------------------------------------------------------------------- # Things related to the banner and usage #------------------------------------------------------------------------- def _banner1_changed(self): self.compute_banner() def _banner2_changed(self): self.compute_banner() def _term_title_changed(self, name, new_value): self.init_term_title() def init_banner(self, banner1, banner2, display_banner): if banner1 is not None: self.banner1 = banner1 if banner2 is not None: self.banner2 = banner2 if display_banner is not None: self.display_banner = display_banner self.compute_banner() def show_banner(self, banner=None): if banner is None: banner = self.banner self.write(banner) def compute_banner(self): self.banner = self.banner1 if self.profile and self.profile != 'default': self.banner += '\nIPython profile: %s\n' % self.profile if self.banner2: self.banner += '\n' + self.banner2 def init_usage(self, usage=None): if usage is None: self.usage = interactive_usage else: self.usage = usage #------------------------------------------------------------------------- # Mainloop and code execution logic #------------------------------------------------------------------------- def mainloop(self, display_banner=None): """Start the mainloop. If an optional banner argument is given, it will override the internally created default banner. """ with nested(self.builtin_trap, self.display_trap): while 1: try: self.interact(display_banner=display_banner) #self.interact_with_readline() # XXX for testing of a readline-decoupled repl loop, call # interact_with_readline above break except KeyboardInterrupt: # this should not be necessary, but KeyboardInterrupt # handling seems rather unpredictable... self.write("\nKeyboardInterrupt in interact()\n") def _replace_rlhist_multiline(self, source_raw, hlen_before_cell): """Store multiple lines as a single entry in history""" # do nothing without readline or disabled multiline if not self.has_readline or not self.multiline_history: return hlen_before_cell # windows rl has no remove_history_item if not hasattr(self.readline, "remove_history_item"): return hlen_before_cell # skip empty cells if not source_raw.rstrip(): return hlen_before_cell # nothing changed do nothing, e.g. when rl removes consecutive dups hlen = self.readline.get_current_history_length() if hlen == hlen_before_cell: return hlen_before_cell for i in range(hlen - hlen_before_cell): self.readline.remove_history_item(hlen - i - 1) stdin_encoding = get_stream_enc(sys.stdin, 'utf-8') self.readline.add_history( py3compat.unicode_to_str(source_raw.rstrip(), stdin_encoding)) return self.readline.get_current_history_length() def interact(self, display_banner=None): """Closely emulate the interactive Python console.""" # batch run -> do not interact if self.exit_now: return if display_banner is None: display_banner = self.display_banner if isinstance(display_banner, basestring): self.show_banner(display_banner) elif display_banner: self.show_banner() more = False if self.has_readline: self.readline_startup_hook(self.pre_readline) hlen_b4_cell = self.readline.get_current_history_length() else: hlen_b4_cell = 0 # exit_now is set by a call to %Exit or %Quit, through the # ask_exit callback. while not self.exit_now: self.hooks.pre_prompt_hook() if more: try: prompt = self.prompt_manager.render('in2') except: self.showtraceback() if self.autoindent: self.rl_do_indent = True else: try: prompt = self.separate_in + self.prompt_manager.render( 'in') except: self.showtraceback() try: line = self.raw_input(prompt) if self.exit_now: # quick exit on sys.std[in|out] close break if self.autoindent: self.rl_do_indent = False except KeyboardInterrupt: #double-guard against keyboardinterrupts during kbdint handling try: self.write('\nKeyboardInterrupt\n') source_raw = self.input_splitter.source_raw_reset()[1] hlen_b4_cell = \ self._replace_rlhist_multiline(source_raw, hlen_b4_cell) more = False except KeyboardInterrupt: pass except EOFError: if self.autoindent: self.rl_do_indent = False if self.has_readline: self.readline_startup_hook(None) self.write('\n') self.exit() except bdb.BdbQuit: warn( 'The Python debugger has exited with a BdbQuit exception.\n' 'Because of how pdb handles the stack, it is impossible\n' 'for IPython to properly format this particular exception.\n' 'IPython will resume normal operation.') except: # exceptions here are VERY RARE, but they can be triggered # asynchronously by signal handlers, for example. self.showtraceback() else: self.input_splitter.push(line) more = self.input_splitter.push_accepts_more() if (self.SyntaxTB.last_syntax_error and self.autoedit_syntax): self.edit_syntax_error() if not more: source_raw = self.input_splitter.source_raw_reset()[1] self.run_cell(source_raw, store_history=True) hlen_b4_cell = \ self._replace_rlhist_multiline(source_raw, hlen_b4_cell) # Turn off the exit flag, so the mainloop can be restarted if desired self.exit_now = False def raw_input(self, prompt=''): """Write a prompt and read a line. The returned line does not include the trailing newline. When the user enters the EOF key sequence, EOFError is raised. Optional inputs: - prompt(''): a string to be printed to prompt the user. - continue_prompt(False): whether this line is the first one or a continuation in a sequence of inputs. """ # Code run by the user may have modified the readline completer state. # We must ensure that our completer is back in place. if self.has_readline: self.set_readline_completer() # raw_input expects str, but we pass it unicode sometimes prompt = py3compat.cast_bytes_py2(prompt) try: line = py3compat.str_to_unicode(self.raw_input_original(prompt)) except ValueError: warn("\n********\nYou or a %run:ed script called sys.stdin.close()" " or sys.stdout.close()!\nExiting IPython!\n") self.ask_exit() return "" # Try to be reasonably smart about not re-indenting pasted input more # than necessary. We do this by trimming out the auto-indent initial # spaces, if the user's actual input started itself with whitespace. if self.autoindent: if num_ini_spaces(line) > self.indent_current_nsp: line = line[self.indent_current_nsp:] self.indent_current_nsp = 0 return line #------------------------------------------------------------------------- # Methods to support auto-editing of SyntaxErrors. #------------------------------------------------------------------------- def edit_syntax_error(self): """The bottom half of the syntax error handler called in the main loop. Loop until syntax error is fixed or user cancels. """ while self.SyntaxTB.last_syntax_error: # copy and clear last_syntax_error err = self.SyntaxTB.clear_err_state() if not self._should_recompile(err): return try: # may set last_syntax_error again if a SyntaxError is raised self.safe_execfile(err.filename, self.user_ns) except: self.showtraceback() else: try: f = open(err.filename) try: # This should be inside a display_trap block and I # think it is. sys.displayhook(f.read()) finally: f.close() except: self.showtraceback() def _should_recompile(self, e): """Utility routine for edit_syntax_error""" if e.filename in ('<ipython console>', '<input>', '<string>', '<console>', '<BackgroundJob compilation>', None): return False try: if (self.autoedit_syntax and not self.ask_yes_no( 'Return to editor to correct syntax error? ' '[Y/n] ', 'y')): return False except EOFError: return False def int0(x): try: return int(x) except TypeError: return 0 # always pass integer line and offset values to editor hook try: self.hooks.fix_error_editor(e.filename, int0(e.lineno), int0(e.offset), e.msg) except TryNext: warn('Could not open editor') return False return True #------------------------------------------------------------------------- # Things related to exiting #------------------------------------------------------------------------- def ask_exit(self): """ Ask the shell to exit. Can be overiden and used as a callback. """ self.exit_now = True def exit(self): """Handle interactive exit. This method calls the ask_exit callback.""" if self.confirm_exit: if self.ask_yes_no('Do you really want to exit ([y]/n)?', 'y'): self.ask_exit() else: self.ask_exit() #------------------------------------------------------------------------- # Things related to magics #------------------------------------------------------------------------- def init_magics(self): super(TerminalInteractiveShell, self).init_magics() self.register_magics(TerminalMagics) def showindentationerror(self): super(TerminalInteractiveShell, self).showindentationerror() if not self.using_paste_magics: print("If you want to paste code into IPython, try the " "%paste and %cpaste magic functions.")
class MongoDB(BaseDB): """MongoDB TaskRecord backend.""" connection_args = List( config=True, help="""Positional arguments to be passed to pymongo.Connection. Only necessary if the default mongodb configuration does not point to your mongod instance.""") connection_kwargs = Dict( config=True, help="""Keyword arguments to be passed to pymongo.Connection. Only necessary if the default mongodb configuration does not point to your mongod instance.""") database = Unicode( "ipython-tasks", config=True, help= """The MongoDB database name to use for storing tasks for this session. If unspecified, a new database will be created with the Hub's IDENT. Specifying the database will result in tasks from previous sessions being available via Clients' db_query and get_result methods.""") _connection = Instance(Connection) # pymongo connection def __init__(self, **kwargs): super(MongoDB, self).__init__(**kwargs) if self._connection is None: self._connection = Connection(*self.connection_args, **self.connection_kwargs) if not self.database: self.database = self.session self._db = self._connection[self.database] self._records = self._db['task_records'] self._records.ensure_index('msg_id', unique=True) self._records.ensure_index('submitted') # for sorting history # for rec in self._records.find def _binary_buffers(self, rec): for key in ('buffers', 'result_buffers'): if rec.get(key, None): rec[key] = map(Binary, rec[key]) return rec def add_record(self, msg_id, rec): """Add a new Task Record, by msg_id.""" # print rec rec = self._binary_buffers(rec) self._records.insert(rec) def get_record(self, msg_id): """Get a specific Task Record, by msg_id.""" r = self._records.find_one({'msg_id': msg_id}) if not r: # r will be '' if nothing is found raise KeyError(msg_id) return r def update_record(self, msg_id, rec): """Update the data in an existing record.""" rec = self._binary_buffers(rec) self._records.update({'msg_id': msg_id}, {'$set': rec}) def drop_matching_records(self, check): """Remove a record from the DB.""" self._records.remove(check) def drop_record(self, msg_id): """Remove a record from the DB.""" self._records.remove({'msg_id': msg_id}) def find_records(self, check, keys=None): """Find records matching a query dict, optionally extracting subset of keys. Returns list of matching records. Parameters ---------- check: dict mongodb-style query argument keys: list of strs [optional] if specified, the subset of keys to extract. msg_id will *always* be included. """ if keys and 'msg_id' not in keys: keys.append('msg_id') matches = list(self._records.find(check, keys)) for rec in matches: rec.pop('_id') return matches def get_history(self): """get all msg_ids, ordered by time submitted.""" cursor = self._records.find({}, {'msg_id': 1}).sort('submitted') return [rec['msg_id'] for rec in cursor]