class Dock(Box):
"""Displays multiple widgets in a dock"""
_view_name = Unicode('DockView', sync=True)
description = Unicode(sync=True)
class IPythonWidget(FrontendWidget):
""" A FrontendWidget for an IPython kernel.
"""
# If set, the 'custom_edit_requested(str, int)' signal will be emitted when
# an editor is needed for a file. This overrides 'editor' and 'editor_line'
# settings.
custom_edit = Bool(False)
custom_edit_requested = QtCore.Signal(object, object)
editor = Unicode(default_editor, config=True,
help="""
A command for invoking a system text editor. If the string contains a
{filename} format specifier, it will be used. Otherwise, the filename
will be appended to the end the command.
""")
editor_line = Unicode(config=True,
help="""
The editor command to use when a specific line number is requested. The
string should contain two format specifiers: {line} and {filename}. If
this parameter is not specified, the line number option to the %edit
magic will be ignored.
""")
style_sheet = Unicode(config=True,
help="""
A CSS stylesheet. The stylesheet can contain classes for:
1. Qt: QPlainTextEdit, QFrame, QWidget, etc
2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter)
3. IPython: .error, .in-prompt, .out-prompt, etc
""")
syntax_style = Unicode(config=True,
help="""
If not empty, use this Pygments style for syntax highlighting.
Otherwise, the style sheet is queried for Pygments style
information.
""")
# Prompts.
in_prompt = Unicode(default_in_prompt, config=True)
out_prompt = Unicode(default_out_prompt, config=True)
input_sep = Unicode(default_input_sep, config=True)
output_sep = Unicode(default_output_sep, config=True)
output_sep2 = Unicode(default_output_sep2, config=True)
# FrontendWidget protected class variables.
_input_splitter_class = IPythonInputSplitter
_prompt_transformer = IPythonInputSplitter(physical_line_transforms=[ipy_prompt()],
logical_line_transforms=[],
python_line_transforms=[],
)
# IPythonWidget protected class variables.
_PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number'])
_payload_source_edit = 'edit_magic'
_payload_source_exit = 'ask_exit'
_payload_source_next_input = 'set_next_input'
_payload_source_page = 'page'
_retrying_history_request = False
_starting = False
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(IPythonWidget, self).__init__(*args, **kw)
# IPythonWidget protected variables.
self._payload_handlers = {
self._payload_source_edit : self._handle_payload_edit,
self._payload_source_exit : self._handle_payload_exit,
self._payload_source_page : self._handle_payload_page,
self._payload_source_next_input : self._handle_payload_next_input }
self._previous_prompt_obj = None
self._keep_kernel_on_exit = None
# Initialize widget styling.
if self.style_sheet:
self._style_sheet_changed()
self._syntax_style_changed()
else:
self.set_default_style()
self._guiref_loaded = False
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_complete_reply(self, rep):
""" Reimplemented to support IPython's improved completion machinery.
"""
self.log.debug("complete: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('complete')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
content = rep['content']
matches = content['matches']
start = content['cursor_start']
end = content['cursor_end']
offset = end - start
# Move the cursor to the start of the match and complete.
cursor.movePosition(QtGui.QTextCursor.Left, n=offset)
self._complete_with_items(cursor, matches)
def _handle_execute_reply(self, msg):
""" Reimplemented to support prompt requests.
"""
msg_id = msg['parent_header'].get('msg_id')
info = self._request_info['execute'].get(msg_id)
if info and info.kind == 'prompt':
content = msg['content']
if content['status'] == 'aborted':
self._show_interpreter_prompt()
else:
number = content['execution_count'] + 1
self._show_interpreter_prompt(number)
self._request_info['execute'].pop(msg_id)
else:
super(IPythonWidget, self)._handle_execute_reply(msg)
def _handle_history_reply(self, msg):
""" Implemented to handle history tail replies, which are only supported
by the IPython kernel.
"""
content = msg['content']
if 'history' not in content:
self.log.error("History request failed: %r"%content)
if content.get('status', '') == 'aborted' and \
not self._retrying_history_request:
# a *different* action caused this request to be aborted, so
# we should try again.
self.log.error("Retrying aborted history request")
# prevent multiple retries of aborted requests:
self._retrying_history_request = True
# wait out the kernel's queue flush, which is currently timed at 0.1s
time.sleep(0.25)
self.kernel_client.shell_channel.history(hist_access_type='tail',n=1000)
else:
self._retrying_history_request = False
return
# reset retry flag
self._retrying_history_request = False
history_items = content['history']
self.log.debug("Received history reply with %i entries", len(history_items))
items = []
last_cell = u""
for _, _, cell in history_items:
cell = cell.rstrip()
if cell != last_cell:
items.append(cell)
last_cell = cell
self._set_history(items)
def _handle_execute_result(self, msg):
""" Reimplemented for IPython-style "display hook".
"""
self.log.debug("execute_result: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
self.flush_clearoutput()
content = msg['content']
prompt_number = content.get('execution_count', 0)
data = content['data']
if 'text/plain' in data:
self._append_plain_text(self.output_sep, True)
self._append_html(self._make_out_prompt(prompt_number), True)
text = data['text/plain']
# If the repr is multiline, make sure we start on a new line,
# so that its lines are aligned.
if "\n" in text and not self.output_sep.endswith("\n"):
self._append_plain_text('\n', True)
self._append_plain_text(text + self.output_sep2, True)
def _handle_display_data(self, msg):
""" The base handler for the ``display_data`` message.
"""
self.log.debug("display: %s", msg.get('content', ''))
# For now, we don't display data from other frontends, but we
# eventually will as this allows all frontends to monitor the display
# data. But we need to figure out how to handle this in the GUI.
if not self._hidden and self._is_from_this_session(msg):
self.flush_clearoutput()
data = msg['content']['data']
metadata = msg['content']['metadata']
# In the regular IPythonWidget, we simply print the plain text
# representation.
if 'text/plain' in data:
text = data['text/plain']
self._append_plain_text(text, True)
# This newline seems to be needed for text and html output.
self._append_plain_text(u'\n', True)
def _handle_kernel_info_reply(self, rep):
"""Handle kernel info replies."""
content = rep['content']
if not self._guiref_loaded:
if content.get('language') == 'python':
self._load_guiref_magic()
self._guiref_loaded = True
self.kernel_banner = content.get('banner', '')
if self._starting:
# finish handling started channels
self._starting = False
super(IPythonWidget, self)._started_channels()
def _started_channels(self):
"""Reimplemented to make a history request and load %guiref."""
self._starting = True
# The reply will trigger %guiref load provided language=='python'
self.kernel_client.kernel_info()
self.kernel_client.shell_channel.history(hist_access_type='tail',
n=1000)
def _load_guiref_magic(self):
"""Load %guiref magic."""
self.kernel_client.shell_channel.execute('\n'.join([
"try:",
" _usage",
"except:",
" from IPython.core import usage as _usage",
" get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')",
" del _usage",
]), silent=True)
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def execute_file(self, path, hidden=False):
""" Reimplemented to use the 'run' magic.
"""
# Use forward slashes on Windows to avoid escaping each separator.
if sys.platform == 'win32':
path = os.path.normpath(path).replace('\\', '/')
# Perhaps we should not be using %run directly, but while we
# are, it is necessary to quote or escape filenames containing spaces
# or quotes.
# In earlier code here, to minimize escaping, we sometimes quoted the
# filename with single quotes. But to do this, this code must be
# platform-aware, because run uses shlex rather than python string
# parsing, so that:
# * In Win: single quotes can be used in the filename without quoting,
# and we cannot use single quotes to quote the filename.
# * In *nix: we can escape double quotes in a double quoted filename,
# but can't escape single quotes in a single quoted filename.
# So to keep this code non-platform-specific and simple, we now only
# use double quotes to quote filenames, and escape when needed:
if ' ' in path or "'" in path or '"' in path:
path = '"%s"' % path.replace('"', '\\"')
self.execute('%%run %s' % path, hidden=hidden)
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _process_execute_error(self, msg):
""" Reimplemented for IPython-style traceback formatting.
"""
content = msg['content']
traceback = '\n'.join(content['traceback']) + '\n'
if False:
# FIXME: For now, tracebacks come as plain text, so we can't use
# the html renderer yet. Once we refactor ultratb to produce
# properly styled tracebacks, this branch should be the default
traceback = traceback.replace(' ', ' ')
traceback = traceback.replace('\n', '<br/>')
ename = content['ename']
ename_styled = '<span class="error">%s</span>' % ename
traceback = traceback.replace(ename, ename_styled)
self._append_html(traceback)
else:
# This is the fallback for now, using plain text with ansi escapes
self._append_plain_text(traceback)
def _process_execute_payload(self, item):
""" Reimplemented to dispatch payloads to handler methods.
"""
handler = self._payload_handlers.get(item['source'])
if handler is None:
# We have no handler for this type of payload, simply ignore it
return False
else:
handler(item)
return True
def _show_interpreter_prompt(self, number=None):
""" Reimplemented for IPython-style prompts.
"""
# If a number was not specified, make a prompt number request.
if number is None:
msg_id = self.kernel_client.shell_channel.execute('', silent=True)
info = self._ExecutionRequest(msg_id, 'prompt')
self._request_info['execute'][msg_id] = info
return
# Show a new prompt and save information about it so that it can be
# updated later if the prompt number turns out to be wrong.
self._prompt_sep = self.input_sep
self._show_prompt(self._make_in_prompt(number), html=True)
block = self._control.document().lastBlock()
length = len(self._prompt)
self._previous_prompt_obj = self._PromptBlock(block, length, number)
# Update continuation prompt to reflect (possibly) new prompt length.
self._set_continuation_prompt(
self._make_continuation_prompt(self._prompt), html=True)
def _show_interpreter_prompt_for_reply(self, msg):
""" Reimplemented for IPython-style prompts.
"""
# Update the old prompt number if necessary.
content = msg['content']
# abort replies do not have any keys:
if content['status'] == 'aborted':
if self._previous_prompt_obj:
previous_prompt_number = self._previous_prompt_obj.number
else:
previous_prompt_number = 0
else:
previous_prompt_number = content['execution_count']
if self._previous_prompt_obj and \
self._previous_prompt_obj.number != previous_prompt_number:
block = self._previous_prompt_obj.block
# Make sure the prompt block has not been erased.
if block.isValid() and block.text():
# Remove the old prompt and insert a new prompt.
cursor = QtGui.QTextCursor(block)
cursor.movePosition(QtGui.QTextCursor.Right,
QtGui.QTextCursor.KeepAnchor,
self._previous_prompt_obj.length)
prompt = self._make_in_prompt(previous_prompt_number)
self._prompt = self._insert_html_fetching_plain_text(
cursor, prompt)
# When the HTML is inserted, Qt blows away the syntax
# highlighting for the line, so we need to rehighlight it.
self._highlighter.rehighlightBlock(cursor.block())
self._previous_prompt_obj = None
# Show a new prompt with the kernel's estimated prompt number.
self._show_interpreter_prompt(previous_prompt_number + 1)
#---------------------------------------------------------------------------
# 'IPythonWidget' interface
#---------------------------------------------------------------------------
def set_default_style(self, colors='lightbg'):
""" Sets the widget style to the class defaults.
Parameters
----------
colors : str, optional (default lightbg)
Whether to use the default IPython light background or dark
background or B&W style.
"""
colors = colors.lower()
if colors=='lightbg':
self.style_sheet = styles.default_light_style_sheet
self.syntax_style = styles.default_light_syntax_style
elif colors=='linux':
self.style_sheet = styles.default_dark_style_sheet
self.syntax_style = styles.default_dark_syntax_style
elif colors=='nocolor':
self.style_sheet = styles.default_bw_style_sheet
self.syntax_style = styles.default_bw_syntax_style
else:
raise KeyError("No such color scheme: %s"%colors)
#---------------------------------------------------------------------------
# 'IPythonWidget' protected interface
#---------------------------------------------------------------------------
def _edit(self, filename, line=None):
""" Opens a Python script for editing.
Parameters
----------
filename : str
A path to a local system file.
line : int, optional
A line of interest in the file.
"""
if self.custom_edit:
self.custom_edit_requested.emit(filename, line)
elif not self.editor:
self._append_plain_text('No default editor available.\n'
'Specify a GUI text editor in the `IPythonWidget.editor` '
'configurable to enable the %edit magic')
else:
try:
filename = '"%s"' % filename
if line and self.editor_line:
command = self.editor_line.format(filename=filename,
line=line)
else:
try:
command = self.editor.format()
except KeyError:
command = self.editor.format(filename=filename)
else:
command += ' ' + filename
except KeyError:
self._append_plain_text('Invalid editor command.\n')
else:
try:
Popen(command, shell=True)
except OSError:
msg = 'Opening editor with command "%s" failed.\n'
self._append_plain_text(msg % command)
def _make_in_prompt(self, number):
""" Given a prompt number, returns an HTML In prompt.
"""
try:
body = self.in_prompt % number
except TypeError:
# allow in_prompt to leave out number, e.g. '>>> '
body = self.in_prompt
return '<span class="in-prompt">%s</span>' % body
def _make_continuation_prompt(self, prompt):
""" Given a plain text version of an In prompt, returns an HTML
continuation prompt.
"""
end_chars = '...: '
space_count = len(prompt.lstrip('\n')) - len(end_chars)
body = ' ' * space_count + end_chars
return '<span class="in-prompt">%s</span>' % body
def _make_out_prompt(self, number):
""" Given a prompt number, returns an HTML Out prompt.
"""
body = self.out_prompt % number
return '<span class="out-prompt">%s</span>' % body
#------ Payload handlers --------------------------------------------------
# Payload handlers with a generic interface: each takes the opaque payload
# dict, unpacks it and calls the underlying functions with the necessary
# arguments.
def _handle_payload_edit(self, item):
self._edit(item['filename'], item['line_number'])
def _handle_payload_exit(self, item):
self._keep_kernel_on_exit = item['keepkernel']
self.exit_requested.emit(self)
def _handle_payload_next_input(self, item):
self.input_buffer = item['text']
def _handle_payload_page(self, item):
# Since the plain text widget supports only a very small subset of HTML
# and we have no control over the HTML source, we only page HTML
# payloads in the rich text widget.
data = item['data']
if 'text/html' in data and self.kind == 'rich':
self._page(data['text/html'], html=True)
else:
self._page(data['text/plain'], html=False)
#------ Trait change handlers --------------------------------------------
def _style_sheet_changed(self):
""" Set the style sheets of the underlying widgets.
"""
self.setStyleSheet(self.style_sheet)
if self._control is not None:
self._control.document().setDefaultStyleSheet(self.style_sheet)
bg_color = self._control.palette().window().color()
self._ansi_processor.set_background_color(bg_color)
if self._page_control is not None:
self._page_control.document().setDefaultStyleSheet(self.style_sheet)
def _syntax_style_changed(self):
""" Set the style for the syntax highlighter.
"""
if self._highlighter is None:
# ignore premature calls
return
if self.syntax_style:
self._highlighter.set_style(self.syntax_style)
else:
self._highlighter.set_style_sheet(self.style_sheet)
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
return "IPython QtConsole {version}\n".format(version=version)
class HistoryManager(HistoryAccessor):
"""A class to organize all history-related functionality in one place.
"""
# Public interface
# An instance of the IPython shell we are attached to
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
# Lists to hold processed and raw history. These start with a blank entry
# so that we can index them starting from 1
input_hist_parsed = List([""])
input_hist_raw = List([""])
# A list of directories visited during session
dir_hist = List()
def _dir_hist_default(self):
try:
return [py3compat.getcwd()]
except OSError:
return []
# A dict of output history, keyed with ints from the shell's
# execution count.
output_hist = Dict()
# The text/plain repr of outputs.
output_hist_reprs = Dict()
# The number of the current session in the history database
session_number = Integer()
db_log_output = Bool(
False,
config=True,
help="Should the history database include output? (default: no)")
db_cache_size = Integer(
0,
config=True,
help=
"Write to database every x commands (higher values save disk access & power).\n"
"Values of 1 or less effectively disable caching.")
# The input and output caches
db_input_cache = List()
db_output_cache = List()
# History saving in separate thread
save_thread = Instance('IPython.core.history.HistorySavingThread')
try: # Event is a function returning an instance of _Event...
save_flag = Instance(threading._Event)
except AttributeError: # ...until Python 3.3, when it's a class.
save_flag = Instance(threading.Event)
# Private interface
# Variables used to store the three last inputs from the user. On each new
# history update, we populate the user's namespace with these, shifted as
# necessary.
_i00 = Unicode(u'')
_i = Unicode(u'')
_ii = Unicode(u'')
_iii = Unicode(u'')
# A regex matching all forms of the exit command, so that we don't store
# them in the history (it's annoying to rewind the first entry and land on
# an exit call).
_exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
def __init__(self, shell=None, config=None, **traits):
"""Create a new history manager associated with a shell instance.
"""
# We need a pointer back to the shell for various tasks.
super(HistoryManager, self).__init__(shell=shell,
config=config,
**traits)
self.save_flag = threading.Event()
self.db_input_cache_lock = threading.Lock()
self.db_output_cache_lock = threading.Lock()
if self.enabled and self.hist_file != ':memory:':
self.save_thread = HistorySavingThread(self)
self.save_thread.start()
self.new_session()
def _get_hist_file_name(self, profile=None):
"""Get default history file name based on the Shell's profile.
The profile parameter is ignored, but must exist for compatibility with
the parent class."""
profile_dir = self.shell.profile_dir.location
return os.path.join(profile_dir, 'history.sqlite')
@needs_sqlite
def new_session(self, conn=None):
"""Get a new session number."""
if conn is None:
conn = self.db
with conn:
cur = conn.execute(
"""INSERT INTO sessions VALUES (NULL, ?, NULL,
NULL, "") """, (datetime.datetime.now(), ))
self.session_number = cur.lastrowid
def end_session(self):
"""Close the database session, filling in the end time and line count."""
self.writeout_cache()
with self.db:
self.db.execute(
"""UPDATE sessions SET end=?, num_cmds=? WHERE
session==?""",
(datetime.datetime.now(), len(self.input_hist_parsed) - 1,
self.session_number))
self.session_number = 0
def name_session(self, name):
"""Give the current session a name in the history database."""
with self.db:
self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
(name, self.session_number))
def reset(self, new_session=True):
"""Clear the session history, releasing all object references, and
optionally open a new session."""
self.output_hist.clear()
# The directory history can't be completely empty
self.dir_hist[:] = [py3compat.getcwd()]
if new_session:
if self.session_number:
self.end_session()
self.input_hist_parsed[:] = [""]
self.input_hist_raw[:] = [""]
self.new_session()
# ------------------------------
# Methods for retrieving history
# ------------------------------
def get_session_info(self, session=0):
"""Get info about a session.
Parameters
----------
session : int
Session number to retrieve. The current session is 0, and negative
numbers count back from current session, so -1 is the previous session.
Returns
-------
session_id : int
Session ID number
start : datetime
Timestamp for the start of the session.
end : datetime
Timestamp for the end of the session, or None if IPython crashed.
num_cmds : int
Number of commands run, or None if IPython crashed.
remark : unicode
A manually set description.
"""
if session <= 0:
session += self.session_number
return super(HistoryManager, self).get_session_info(session=session)
def _get_range_session(self, start=1, stop=None, raw=True, output=False):
"""Get input and output history from the current session. Called by
get_range, and takes similar parameters."""
input_hist = self.input_hist_raw if raw else self.input_hist_parsed
n = len(input_hist)
if start < 0:
start += n
if not stop or (stop > n):
stop = n
elif stop < 0:
stop += n
for i in range(start, stop):
if output:
line = (input_hist[i], self.output_hist_reprs.get(i))
else:
line = input_hist[i]
yield (0, i, line)
def get_range(self, session=0, start=1, stop=None, raw=True, output=False):
"""Retrieve input by session.
Parameters
----------
session : int
Session number to retrieve. The current session is 0, and negative
numbers count back from current session, so -1 is previous session.
start : int
First line to retrieve.
stop : int
End of line range (excluded from output itself). If None, retrieve
to the end of the session.
raw : bool
If True, return untranslated input
output : bool
If True, attempt to include output. This will be 'real' Python
objects for the current session, or text reprs from previous
sessions if db_log_output was enabled at the time. Where no output
is found, None is used.
Returns
-------
entries
An iterator over the desired lines. Each line is a 3-tuple, either
(session, line, input) if output is False, or
(session, line, (input, output)) if output is True.
"""
if session <= 0:
session += self.session_number
if session == self.session_number: # Current session
return self._get_range_session(start, stop, raw, output)
return super(HistoryManager, self).get_range(session, start, stop, raw,
output)
## ----------------------------
## Methods for storing history:
## ----------------------------
def store_inputs(self, line_num, source, source_raw=None):
"""Store source and raw input in history and create input cache
variables ``_i*``.
Parameters
----------
line_num : int
The prompt number of this input.
source : str
Python input.
source_raw : str, optional
If given, this is the raw input without any IPython transformations
applied to it. If not given, ``source`` is used.
"""
if source_raw is None:
source_raw = source
source = source.rstrip('\n')
source_raw = source_raw.rstrip('\n')
# do not store exit/quit commands
if self._exit_re.match(source_raw.strip()):
return
self.input_hist_parsed.append(source)
self.input_hist_raw.append(source_raw)
with self.db_input_cache_lock:
self.db_input_cache.append((line_num, source, source_raw))
# Trigger to flush cache and write to DB.
if len(self.db_input_cache) >= self.db_cache_size:
self.save_flag.set()
# update the auto _i variables
self._iii = self._ii
self._ii = self._i
self._i = self._i00
self._i00 = source_raw
# hackish access to user namespace to create _i1,_i2... dynamically
new_i = '_i%s' % line_num
to_main = {
'_i': self._i,
'_ii': self._ii,
'_iii': self._iii,
new_i: self._i00
}
if self.shell is not None:
self.shell.push(to_main, interactive=False)
def store_output(self, line_num):
"""If database output logging is enabled, this saves all the
outputs from the indicated prompt number to the database. It's
called by run_cell after code has been executed.
Parameters
----------
line_num : int
The line number from which to save outputs
"""
if (not self.db_log_output) or (line_num
not in self.output_hist_reprs):
return
output = self.output_hist_reprs[line_num]
with self.db_output_cache_lock:
self.db_output_cache.append((line_num, output))
if self.db_cache_size <= 1:
self.save_flag.set()
def _writeout_input_cache(self, conn):
with conn:
for line in self.db_input_cache:
conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
(self.session_number, ) + line)
def _writeout_output_cache(self, conn):
with conn:
for line in self.db_output_cache:
conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
(self.session_number, ) + line)
@needs_sqlite
def writeout_cache(self, conn=None):
"""Write any entries in the cache to the database."""
if conn is None:
conn = self.db
with self.db_input_cache_lock:
try:
self._writeout_input_cache(conn)
except sqlite3.IntegrityError:
self.new_session(conn)
print("ERROR! Session/line number was not unique in",
"database. History logging moved to new session",
self.session_number)
try:
# Try writing to the new session. If this fails, don't
# recurse
self._writeout_input_cache(conn)
except sqlite3.IntegrityError:
pass
finally:
self.db_input_cache = []
with self.db_output_cache_lock:
try:
self._writeout_output_cache(conn)
except sqlite3.IntegrityError:
print("!! Session/line number for output was not unique",
"in database. Output will not be stored.")
finally:
self.db_output_cache = []
class AutoHandler(PrefilterHandler):
handler_name = Unicode('auto')
esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
def handle(self, line_info):
"""Handle lines which can be auto-executed, quoting if requested."""
line = line_info.line
ifun = line_info.ifun
the_rest = line_info.the_rest
pre = line_info.pre
esc = line_info.esc
continue_prompt = line_info.continue_prompt
obj = line_info.ofind(self.shell)['obj']
#print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest) # dbg
# This should only be active for single-line input!
if continue_prompt:
return line
force_auto = isinstance(obj, IPyAutocall)
# User objects sometimes raise exceptions on attribute access other
# than AttributeError (we've seen it in the past), so it's safest to be
# ultra-conservative here and catch all.
try:
auto_rewrite = obj.rewrite
except Exception:
auto_rewrite = True
if esc == ESC_QUOTE:
# Auto-quote splitting on whitespace
newcmd = '%s("%s")' % (ifun, '", "'.join(the_rest.split()))
elif esc == ESC_QUOTE2:
# Auto-quote whole string
newcmd = '%s("%s")' % (ifun, the_rest)
elif esc == ESC_PAREN:
newcmd = '%s(%s)' % (ifun, ",".join(the_rest.split()))
else:
# Auto-paren.
if force_auto:
# Don't rewrite if it is already a call.
do_rewrite = not the_rest.startswith('(')
else:
if not the_rest:
# We only apply it to argument-less calls if the autocall
# parameter is set to 2.
do_rewrite = (self.shell.autocall >= 2)
elif the_rest.startswith('[') and hasattr(obj, '__getitem__'):
# Don't autocall in this case: item access for an object
# which is BOTH callable and implements __getitem__.
do_rewrite = False
else:
do_rewrite = True
# Figure out the rewritten command
if do_rewrite:
if the_rest.endswith(';'):
newcmd = '%s(%s);' % (ifun.rstrip(), the_rest[:-1])
else:
newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
else:
normal_handler = self.prefilter_manager.get_handler_by_name(
'normal')
return normal_handler.handle(line_info)
# Display the rewritten call
if auto_rewrite:
self.shell.auto_rewrite_input(newcmd)
return newcmd
class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp):
name = 'ipython-qtconsole'
description = """
The IPython QtConsole.
This launches a Console-style application using Qt. It is not a full
console, in that launched terminal subprocesses will not be able to accept
input.
The QtConsole supports various extra features beyond the Terminal IPython
shell, such as inline plotting with matplotlib, via:
ipython qtconsole --pylab=inline
as well as saving your session as HTML, and printing the output.
"""
examples = _examples
classes = [IPythonWidget] + IPythonConsoleApp.classes
flags = Dict(flags)
aliases = Dict(aliases)
frontend_flags = Any(qt_flags)
frontend_aliases = Any(qt_aliases)
kernel_client_class = QtKernelClient
kernel_manager_class = QtKernelManager
stylesheet = Unicode('',
config=True,
help="path to a custom CSS stylesheet")
plain = CBool(
False,
config=True,
help=
"Use a plaintext widget instead of rich text (plain can't print/save)."
)
def _plain_changed(self, name, old, new):
kind = 'plain' if new else 'rich'
self.config.ConsoleWidget.kind = kind
if new:
self.widget_factory = IPythonWidget
else:
self.widget_factory = RichIPythonWidget
# the factory for creating a widget
widget_factory = Any(RichIPythonWidget)
def parse_command_line(self, argv=None):
super(IPythonQtConsoleApp, self).parse_command_line(argv)
self.build_kernel_argv(argv)
def new_frontend_master(self):
""" Create and return new frontend attached to new kernel, launched on localhost.
"""
kernel_manager = self.kernel_manager_class(
connection_file=self._new_connection_file(),
config=self.config,
autorestart=True,
)
# start the kernel
kwargs = dict()
kwargs['extra_arguments'] = self.kernel_argv
kernel_manager.start_kernel(**kwargs)
kernel_manager.client_factory = self.kernel_client_class
kernel_client = kernel_manager.client()
kernel_client.start_channels(shell=True, iopub=True)
widget = self.widget_factory(config=self.config, local_kernel=True)
self.init_colors(widget)
widget.kernel_manager = kernel_manager
widget.kernel_client = kernel_client
widget._existing = False
widget._may_close = True
widget._confirm_exit = self.confirm_exit
return widget
def new_frontend_slave(self, current_widget):
"""Create and return a new frontend attached to an existing kernel.
Parameters
----------
current_widget : IPythonWidget
The IPythonWidget whose kernel this frontend is to share
"""
kernel_client = self.kernel_client_class(
connection_file=current_widget.kernel_client.connection_file,
config=self.config,
)
kernel_client.load_connection_file()
kernel_client.start_channels()
widget = self.widget_factory(config=self.config, local_kernel=False)
self.init_colors(widget)
widget._existing = True
widget._may_close = False
widget._confirm_exit = False
widget.kernel_client = kernel_client
widget.kernel_manager = current_widget.kernel_manager
return widget
def init_qt_app(self):
# separate from qt_elements, because it must run first
self.app = QtGui.QApplication([])
def init_qt_elements(self):
# Create the widget.
base_path = os.path.abspath(os.path.dirname(__file__))
icon_path = os.path.join(base_path, 'resources', 'icon',
'IPythonConsole.svg')
self.app.icon = QtGui.QIcon(icon_path)
QtGui.QApplication.setWindowIcon(self.app.icon)
try:
ip = self.config.KernelManager.ip
except AttributeError:
ip = LOCALHOST
local_kernel = (not self.existing) or ip in LOCAL_IPS
self.widget = self.widget_factory(config=self.config,
local_kernel=local_kernel)
self.init_colors(self.widget)
self.widget._existing = self.existing
self.widget._may_close = not self.existing
self.widget._confirm_exit = self.confirm_exit
self.widget.kernel_manager = self.kernel_manager
self.widget.kernel_client = self.kernel_client
self.window = MainWindow(
self.app,
confirm_exit=self.confirm_exit,
new_frontend_factory=self.new_frontend_master,
slave_frontend_factory=self.new_frontend_slave,
)
self.window.log = self.log
self.window.add_tab_with_frontend(self.widget)
self.window.init_menu_bar()
self.window.setWindowTitle('IPython')
def init_colors(self, widget):
"""Configure the coloring of the widget"""
# Note: This will be dramatically simplified when colors
# are removed from the backend.
# parse the colors arg down to current known labels
try:
colors = self.config.ZMQInteractiveShell.colors
except AttributeError:
colors = None
try:
style = self.config.IPythonWidget.syntax_style
except AttributeError:
style = None
try:
sheet = self.config.IPythonWidget.style_sheet
except AttributeError:
sheet = None
# find the value for colors:
if colors:
colors = colors.lower()
if colors in ('lightbg', 'light'):
colors = 'lightbg'
elif colors in ('dark', 'linux'):
colors = 'linux'
else:
colors = 'nocolor'
elif style:
if style == 'bw':
colors = 'nocolor'
elif styles.dark_style(style):
colors = 'linux'
else:
colors = 'lightbg'
else:
colors = None
# Configure the style
if style:
widget.style_sheet = styles.sheet_from_template(style, colors)
widget.syntax_style = style
widget._syntax_style_changed()
widget._style_sheet_changed()
elif colors:
# use a default dark/light/bw style
widget.set_default_style(colors=colors)
if self.stylesheet:
# we got an explicit stylesheet
if os.path.isfile(self.stylesheet):
with open(self.stylesheet) as f:
sheet = f.read()
else:
raise IOError("Stylesheet %r not found." % self.stylesheet)
if sheet:
widget.style_sheet = sheet
widget._style_sheet_changed()
def init_signal(self):
"""allow clean shutdown on sigint"""
signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-2))
# need a timer, so that QApplication doesn't block until a real
# Qt event fires (can require mouse movement)
# timer trick from http://stackoverflow.com/q/4938723/938949
timer = QtCore.QTimer()
# Let the interpreter run each 200 ms:
timer.timeout.connect(lambda: None)
timer.start(200)
# hold onto ref, so the timer doesn't get cleaned up
self._sigint_timer = timer
@catch_config_error
def initialize(self, argv=None):
self.init_qt_app()
super(IPythonQtConsoleApp, self).initialize(argv)
IPythonConsoleApp.initialize(self, argv)
self.init_qt_elements()
self.init_signal()
def start(self):
# draw the window
self.window.show()
self.window.raise_()
# Start the application main loop.
self.app.exec_()
class MetaManager(NBXContentsManager):
"""
Holds NotebookManager classes and routes calls to the appropiate
manager.
"""
debug = Bool(True)
file_dirs = Dict(config=True, help="Dict of alias, path")
bundle_dirs = Dict(config=True,
help="BundleNBManager. Dict of alias, path")
github_accounts = List(
Tuple,
config=True,
help="List of Tuple(github_account, github_password)")
manager_middleware = Dict(config=True, help="Dict of Middleware")
# Not sure if this should be optional. For now, make it configurable
enable_custom_handlers = Bool(True,
config=True,
help="Enable Custom Handlers")
root_dir = Unicode(getcwd())
def __init__(self, *args, **kwargs):
super(MetaManager, self).__init__(*args, **kwargs)
self.app = kwargs['parent']
self.managers = {}
server_home = FileContentsManager()
server_home.root_dir = self.root_dir
self.managers['server-home'] = server_home
if self.enable_custom_handlers:
enable_custom_handlers()
for alias, path in self.file_dirs.items():
fb = FileContentsManager()
fb.root_dir = path
self.managers[alias] = fb
for alias, path in self.bundle_dirs.items():
fb = BundleNotebookManager()
fb.root_dir = path
self.managers[alias] = fb
for user, pw in self.github_accounts:
gh = notebook_gisthub(user, pw)
gbm = GistNotebookManager(gisthub=gh)
self.managers['gist:' + user] = gbm
self.middleware = {}
for name, middleware in self.manager_middleware.items():
cls = import_item(middleware)
self.middleware[name] = cls(parent=self, log=self.log)
self.root = RootManager(meta_manager=self)
def dispatch_middleware(self, hook_name, *args, **kwargs):
"""
dispatch hook calls to middleware
"""
for name, middleware in self.middleware.items():
method = getattr(middleware, hook_name, None)
if method is not None:
method(*args, **kwargs)
def _nbm_from_path(self, path='', name=None):
"""
So this helper function takes in a request path and returns
the manager for that path.
Note: I would like to only take in a single full request path, but
IPython is odd in its handling of this. So I have to accept a name
param. Sometimes the name is really a path. blah
"""
if self.debug:
print('nbm_from_path(path={0}, name={1})'.format(path, name))
import inspect
print(inspect.stack()[1][3])
# TODO clean up this logic.
request_path = self._get_fullpath(name, path)
# remove beginning slash (/)
if request_path and request_path[0] == os.sep:
request_path = request_path[1:]
meta = ManagerMeta()
# we are on root
if not request_path:
meta.request_path = ''
meta.path = ''
meta.name = ''
return self.root, meta
bits = request_path.split(os.sep)
manager_path = bits.pop(0)
# if we were passed in a name we should spit it back out
# to remain compatible with IPython. This really only seems
# to be an issue with get_model
local_name = ''
if name and len(bits) >= 1:
local_name = bits.pop()
local_path = os.sep.join(bits)
meta.request_path = request_path
meta.path = local_path
meta.name = local_name
meta.nbm_path = manager_path
nbm = self.managers.get(manager_path)
if self.debug:
print(nbm, meta)
return nbm, meta
def _get_fullpath(self, name=None, path=''):
if name is not None:
path = url_path_join(path, name)
return path
def list_dirs(self, path):
nbm, meta = self._nbm_from_path(path)
val = nbm.list_dirs(meta.path)
return val
# ContentsManager API part 1
def path_exists(self, path):
nbm, meta = self._nbm_from_path(path)
if nbm is None:
return False
exists = nbm.path_exists(meta.path)
return exists
def is_hidden(self, path):
nbm, meta = self._nbm_from_path(path)
return nbm.is_hidden(meta.path)
def file_exists(self, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.file_exists(meta.name, meta.path)
def exists(self, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
if nbm is None:
return False
exists = nbm.exists(meta.name, meta.path)
return exists
def get_model(self, name, path='', content=True):
nbm, meta = self._nbm_from_path(path, name)
model = nbm.get_model(meta.name, path=meta.path, content=content)
# while the local manager doesn't know its nbm_path,
# we have to add it back in for the metamanager.
if model['type'] == 'directory':
content = model.get("content", [])
for m in content:
m['path'] = meta.request_path
return model
@manager_hook
def save(self, model, name='', path=''):
nbm, meta = self._nbm_from_path(path, name)
# make sure path is local and doesn't include sub manager prefix
model['path'] = meta.path
model = nbm.save(model=model, name=meta.name, path=meta.path)
return model
def update(self, model, name, path=''):
"""Update the notebook and return the model with no content."""
nbm, meta = self._nbm_from_path(path, name)
return nbm.update(model, meta.name, meta.path)
def delete(self, name, path=''):
"""Delete notebook by name and path."""
nbm, meta = self._nbm_from_path(path, name)
return nbm.delete(meta.name, meta.path)
def create_checkpoint(self, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.create_checkpoint(meta.name, meta.path)
def list_checkpoints(self, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.list_checkpoints(meta.name, meta.path)
def restore_checkpoint(self, checkpoint_id, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.restore_checkpoint(checkpoint_id, meta.name, meta.path)
def delete_checkpoint(self, checkpoint_id, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.delete_checkpoint(checkpoint_id, meta.name, meta.path)
# ContentsManager API part 2: methods that have useable default
# implementations, but can be overridden in subclasses.
# Note, some of these just call the default ContentsManager
# implementation. In reality, they won't need to ever be overridden.
def info_string(self):
infos = [nbm.info_string() for nbm in self.managers.values()]
return "\n".join(infos)
def get_kernel_path(self, name, path=''):
""" defined where kernel for notebooks is started """
nbm, meta = self._nbm_from_path(path, name)
return nbm.get_kernel_path(meta.name, meta.path)
def increment_filename(self, filename, path=''):
nbm, meta = self._nbm_from_path(path)
return nbm.get_kernel_path(filename, meta.path)
def create_file(self, model=None, path='', ext='.ipynb'):
nbm, meta = self._nbm_from_path(path)
return nbm.create_file(model, meta.path, ext)
def copy(self, from_name, to_name=None, path=''):
nbm, meta = self._nbm_from_path(path)
model = nbm.copy(from_name, to_name, meta.path)
model['path'] = path
return model
def trust_notebook(self, name, path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.trust_notebook(meta.name, meta.path)
def check_and_sign(self, nb, name='', path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.check_and_sign(nb, meta.name, meta.path)
def mark_trusted_cells(self, nb, name='', path=''):
nbm, meta = self._nbm_from_path(path, name)
return nbm.mark_trusted_cells(nb, meta.name, meta.path)
## END ContentsManager API
# StaticFileHandler
def get_absolute_path(self, path):
""" return absolute path for static file handler """
nbm, meta = self._nbm_from_path(path)
return nbm.get_absolute_path(meta.path)
class TemplateExporter(Exporter):
"""
Exports notebooks into other file formats. Uses Jinja 2 templating engine
to output new formats. Inherit from this class if you are creating a new
template type along with new filters/preprocessors. If the filters/
preprocessors provided by default suffice, there is no need to inherit from
this class. Instead, override the template_file and file_extension
traits via a config file.
{filters}
"""
# finish the docstring
__doc__ = __doc__.format(filters = '- '+'\n - '.join(default_filters.keys()))
template_file = Unicode(u'default',
config=True,
help="Name of the template file to use")
def _template_file_changed(self, name, old, new):
if new == 'default':
self.template_file = self.default_template
else:
self.template_file = new
self.template = None
self._load_template()
default_template = Unicode(u'')
template = Any()
environment = Any()
template_path = List(['.'], config=True)
def _template_path_changed(self, name, old, new):
self._load_template()
default_template_path = Unicode(
os.path.join("..", "templates"),
help="Path where the template files are located.")
template_skeleton_path = Unicode(
os.path.join("..", "templates", "skeleton"),
help="Path where the template skeleton files are located.")
#Jinja block definitions
jinja_comment_block_start = Unicode("", config=True)
jinja_comment_block_end = Unicode("", config=True)
jinja_variable_block_start = Unicode("", config=True)
jinja_variable_block_end = Unicode("", config=True)
jinja_logic_block_start = Unicode("", config=True)
jinja_logic_block_end = Unicode("", config=True)
#Extension that the template files use.
template_extension = Unicode(".tpl", config=True)
filters = Dict(config=True,
help="""Dictionary of filters, by name and namespace, to add to the Jinja
environment.""")
raw_mimetypes = List(config=True,
help="""formats of raw cells to be included in this Exporter's output."""
)
def _raw_mimetypes_default(self):
return [self.output_mimetype, '']
def __init__(self, config=None, extra_loaders=None, **kw):
"""
Public constructor
Parameters
----------
config : config
User configuration instance.
extra_loaders : list[of Jinja Loaders]
ordered list of Jinja loader to find templates. Will be tried in order
before the default FileSystem ones.
template : str (optional, kw arg)
Template to use when exporting.
"""
super(TemplateExporter, self).__init__(config=config, **kw)
#Init
self._init_template()
self._init_environment(extra_loaders=extra_loaders)
self._init_filters()
def _load_template(self):
"""Load the Jinja template object from the template file
This is a no-op if the template attribute is already defined,
or the Jinja environment is not setup yet.
This is triggered by various trait changes that would change the template.
"""
from jinja2 import TemplateNotFound
if self.template is not None:
return
# called too early, do nothing
if self.environment is None:
return
# Try different template names during conversion. First try to load the
# template by name with extension added, then try loading the template
# as if the name is explicitly specified, then try the name as a
# 'flavor', and lastly just try to load the template by module name.
try_names = []
if self.template_file:
try_names.extend([
self.template_file + self.template_extension,
self.template_file,
])
for try_name in try_names:
self.log.debug("Attempting to load template %s", try_name)
try:
self.template = self.environment.get_template(try_name)
except (TemplateNotFound, IOError):
pass
except Exception as e:
self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True)
else:
self.log.info("Loaded template %s", try_name)
break
@docstring_nbformat_mod
def from_notebook_node(self, nb, resources=None, **kw):
"""
Convert a notebook from a notebook node instance.
Parameters
----------
nb : :class:`~{nbformat_mod}.nbbase.NotebookNode`
Notebook node
resources : dict
Additional resources that can be accessed read/write by
preprocessors and filters.
"""
nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
resources.setdefault('raw_mimetypes', self.raw_mimetypes)
self._load_template()
if self.template is not None:
output = self.template.render(nb=nb_copy, resources=resources)
else:
raise IOError('template file "%s" could not be found' % self.template_file)
return output, resources
def register_filter(self, name, jinja_filter):
"""
Register a filter.
A filter is a function that accepts and acts on one string.
The filters are accesible within the Jinja templating engine.
Parameters
----------
name : str
name to give the filter in the Jinja engine
filter : filter
"""
if jinja_filter is None:
raise TypeError('filter')
isclass = isinstance(jinja_filter, type)
constructed = not isclass
#Handle filter's registration based on it's type
if constructed and isinstance(jinja_filter, py3compat.string_types):
#filter is a string, import the namespace and recursively call
#this register_filter method
filter_cls = import_item(jinja_filter)
return self.register_filter(name, filter_cls)
if constructed and hasattr(jinja_filter, '__call__'):
#filter is a function, no need to construct it.
self.environment.filters[name] = jinja_filter
return jinja_filter
elif isclass and isinstance(jinja_filter, MetaHasTraits):
#filter is configurable. Make sure to pass in new default for
#the enabled flag if one was specified.
filter_instance = jinja_filter(parent=self)
self.register_filter(name, filter_instance )
elif isclass:
#filter is not configurable, construct it
filter_instance = jinja_filter()
self.register_filter(name, filter_instance)
else:
#filter is an instance of something without a __call__
#attribute.
raise TypeError('filter')
def _init_template(self):
"""
Make sure a template name is specified. If one isn't specified, try to
build one from the information we know.
"""
self._template_file_changed('template_file', self.template_file, self.template_file)
def _init_environment(self, extra_loaders=None):
"""
Create the Jinja templating environment.
"""
from jinja2 import Environment, ChoiceLoader, FileSystemLoader
here = os.path.dirname(os.path.realpath(__file__))
loaders = []
if extra_loaders:
loaders.extend(extra_loaders)
paths = self.template_path
paths.extend([os.path.join(here, self.default_template_path),
os.path.join(here, self.template_skeleton_path)])
loaders.append(FileSystemLoader(paths))
self.environment = Environment(
loader= ChoiceLoader(loaders),
extensions=JINJA_EXTENSIONS
)
#Set special Jinja2 syntax that will not conflict with latex.
if self.jinja_logic_block_start:
self.environment.block_start_string = self.jinja_logic_block_start
if self.jinja_logic_block_end:
self.environment.block_end_string = self.jinja_logic_block_end
if self.jinja_variable_block_start:
self.environment.variable_start_string = self.jinja_variable_block_start
if self.jinja_variable_block_end:
self.environment.variable_end_string = self.jinja_variable_block_end
if self.jinja_comment_block_start:
self.environment.comment_start_string = self.jinja_comment_block_start
if self.jinja_comment_block_end:
self.environment.comment_end_string = self.jinja_comment_block_end
def _init_filters(self):
"""
Register all of the filters required for the exporter.
"""
#Add default filters to the Jinja2 environment
for key, value in default_filters.items():
self.register_filter(key, value)
#Load user filters. Overwrite existing filters if need be.
if self.filters:
for key, user_filter in self.filters.items():
self.register_filter(key, user_filter)
class BoundedIntTextWidget(_BoundedIntWidget):
_view_name = Unicode('IntTextView', sync=True)
class IntSliderWidget(_BoundedIntWidget):
_view_name = Unicode('IntSliderView', sync=True)
orientation = Enum([u'horizontal', u'vertical'],
u'horizontal',
help="Vertical or horizontal.",
sync=True)
class _IntWidget(DOMWidget):
value = CInt(0, help="Int value", sync=True)
disabled = Bool(False, help="Enable or disable user changes", sync=True)
description = Unicode(
help="Description of the value this widget represents", sync=True)
class IntTextWidget(_IntWidget):
_view_name = Unicode('IntTextView', sync=True)
class FileNotebookManager(NotebookManager):
save_script = Bool(
False,
config=True,
help="""Automatically create a Python script when saving the notebook.
For easier use of import, %run and %load across notebooks, a
<notebook-name>.py script will be created next to any
<notebook-name>.ipynb on each save. This can also be set with the
short `--script` flag.
""")
checkpoint_dir = Unicode(
config=True,
help="""The location in which to keep notebook checkpoints
By default, it is notebook-dir/.ipynb_checkpoints
""")
def _checkpoint_dir_default(self):
return os.path.join(self.notebook_dir, '.ipynb_checkpoints')
def _checkpoint_dir_changed(self, name, old, new):
"""do a bit of validation of the checkpoint dir"""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
abs_new = os.path.abspath(new)
self.checkpoint_dir = abs_new
return
if os.path.exists(new) and not os.path.isdir(new):
raise TraitError("checkpoint dir %r is not a directory" % new)
if not os.path.exists(new):
self.log.info("Creating checkpoint dir %s", new)
try:
os.mkdir(new)
except:
raise TraitError("Couldn't create checkpoint dir %r" % new)
def get_notebook_names(self, path=''):
"""List all notebook names in the notebook dir and path."""
path = path.strip('/')
if not os.path.isdir(self.get_os_path(path=path)):
raise web.HTTPError(404, 'Directory not found: ' + path)
names = glob.glob(self.get_os_path('*' + self.filename_ext, path))
names = [os.path.basename(name) for name in names]
return names
def increment_filename(self, basename, path='', ext='.ipynb'):
"""Return a non-used filename of the form basename<int>."""
path = path.strip('/')
for i in itertools.count():
name = u'{basename}{i}{ext}'.format(basename=basename,
i=i,
ext=ext)
os_path = self.get_os_path(name, path)
if not os.path.isfile(os_path):
break
return name
def path_exists(self, path):
"""Does the API-style path (directory) actually exist?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to base notebook-dir).
Returns
-------
exists : bool
Whether the path is indeed a directory.
"""
path = path.strip('/')
os_path = self.get_os_path(path=path)
return os.path.isdir(os_path)
def get_os_path(self, name=None, path=''):
"""Given a notebook name and a URL path, return its file system
path.
Parameters
----------
name : string
The name of a notebook file with the .ipynb extension
path : string
The relative URL path (with '/' as separator) to the named
notebook.
Returns
-------
path : string
A file system path that combines notebook_dir (location where
server started), the relative path, and the filename with the
current operating system's url.
"""
parts = path.strip('/').split('/')
parts = [p for p in parts if p != ''] # remove duplicate splits
if name is not None:
parts.append(name)
path = os.path.join(self.notebook_dir, *parts)
return path
def notebook_exists(self, name, path=''):
"""Returns a True if the notebook exists. Else, returns False.
Parameters
----------
name : string
The name of the notebook you are checking.
path : string
The relative path to the notebook (with '/' as separator)
Returns
-------
bool
"""
path = path.strip('/')
nbpath = self.get_os_path(name, path=path)
return os.path.isfile(nbpath)
def list_notebooks(self, path):
"""Returns a list of dictionaries that are the standard model
for all notebooks in the relative 'path'.
Parameters
----------
path : str
the URL path that describes the relative path for the
listed notebooks
Returns
-------
notebooks : list of dicts
a list of the notebook models without 'content'
"""
path = path.strip('/')
notebook_names = self.get_notebook_names(path)
notebooks = []
for name in notebook_names:
model = self.get_notebook_model(name, path, content=False)
notebooks.append(model)
notebooks = sorted(notebooks, key=lambda item: item['name'])
return notebooks
def get_notebook_model(self, name, path='', content=True):
""" Takes a path and name for a notebook and returns it's model
Parameters
----------
name : str
the name of the notebook
path : str
the URL path that describes the relative path for
the notebook
Returns
-------
model : dict
the notebook model. If contents=True, returns the 'contents'
dict in the model as well.
"""
path = path.strip('/')
if not self.notebook_exists(name=name, path=path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
os_path = self.get_os_path(name, path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the notebook model.
model = {}
model['name'] = name
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
if content is True:
with io.open(os_path, 'r', encoding='utf-8') as f:
try:
nb = current.read(f, u'json')
except Exception as e:
raise web.HTTPError(
400, u"Unreadable Notebook: %s %s" % (os_path, e))
model['content'] = nb
return model
def save_notebook_model(self, model, name='', path=''):
"""Save the notebook model and return the model with no content."""
path = path.strip('/')
if 'content' not in model:
raise web.HTTPError(400, u'No notebook JSON data provided')
# One checkpoint should always exist
if self.notebook_exists(
name, path) and not self.list_checkpoints(name, path):
self.create_checkpoint(name, path)
new_path = model.get('path', path).strip('/')
new_name = model.get('name', name)
if path != new_path or name != new_name:
self.rename_notebook(name, path, new_name, new_path)
# Save the notebook file
os_path = self.get_os_path(new_name, new_path)
nb = current.to_notebook_json(model['content'])
if 'name' in nb['metadata']:
nb['metadata']['name'] = u''
try:
self.log.debug("Autosaving notebook %s", os_path)
with io.open(os_path, 'w', encoding='utf-8') as f:
current.write(nb, f, u'json')
except Exception as e:
raise web.HTTPError(
400, u'Unexpected error while autosaving notebook: %s %s' %
(os_path, e))
# Save .py script as well
if self.save_script:
py_path = os.path.splitext(os_path)[0] + '.py'
self.log.debug("Writing script %s", py_path)
try:
with io.open(py_path, 'w', encoding='utf-8') as f:
current.write(nb, f, u'py')
except Exception as e:
raise web.HTTPError(
400,
u'Unexpected error while saving notebook as script: %s %s'
% (py_path, e))
model = self.get_notebook_model(new_name, new_path, content=False)
return model
def update_notebook_model(self, model, name, path=''):
"""Update the notebook's path and/or name"""
path = path.strip('/')
new_name = model.get('name', name)
new_path = model.get('path', path).strip('/')
if path != new_path or name != new_name:
self.rename_notebook(name, path, new_name, new_path)
model = self.get_notebook_model(new_name, new_path, content=False)
return model
def delete_notebook_model(self, name, path=''):
"""Delete notebook by name and path."""
path = path.strip('/')
os_path = self.get_os_path(name, path)
if not os.path.isfile(os_path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % os_path)
# clear checkpoints
for checkpoint in self.list_checkpoints(name, path):
checkpoint_id = checkpoint['id']
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if os.path.isfile(cp_path):
self.log.debug("Unlinking checkpoint %s", cp_path)
os.unlink(cp_path)
self.log.debug("Unlinking notebook %s", os_path)
os.unlink(os_path)
def rename_notebook(self, old_name, old_path, new_name, new_path):
"""Rename a notebook."""
old_path = old_path.strip('/')
new_path = new_path.strip('/')
if new_name == old_name and new_path == old_path:
return
new_os_path = self.get_os_path(new_name, new_path)
old_os_path = self.get_os_path(old_name, old_path)
# Should we proceed with the move?
if os.path.isfile(new_os_path):
raise web.HTTPError(
409, u'Notebook with name already exists: %s' % new_os_path)
if self.save_script:
old_py_path = os.path.splitext(old_os_path)[0] + '.py'
new_py_path = os.path.splitext(new_os_path)[0] + '.py'
if os.path.isfile(new_py_path):
raise web.HTTPError(
409, u'Python script with name already exists: %s' %
new_py_path)
# Move the notebook file
try:
os.rename(old_os_path, new_os_path)
except Exception as e:
raise web.HTTPError(
500,
u'Unknown error renaming notebook: %s %s' % (old_os_path, e))
# Move the checkpoints
old_checkpoints = self.list_checkpoints(old_name, old_path)
for cp in old_checkpoints:
checkpoint_id = cp['id']
old_cp_path = self.get_checkpoint_path(checkpoint_id, old_name,
old_path)
new_cp_path = self.get_checkpoint_path(checkpoint_id, new_name,
new_path)
if os.path.isfile(old_cp_path):
self.log.debug("Renaming checkpoint %s -> %s", old_cp_path,
new_cp_path)
os.rename(old_cp_path, new_cp_path)
# Move the .py script
if self.save_script:
os.rename(old_py_path, new_py_path)
# Checkpoint-related utilities
def get_checkpoint_path(self, checkpoint_id, name, path=''):
"""find the path to a checkpoint"""
path = path.strip('/')
basename, _ = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=self.filename_ext,
)
cp_path = os.path.join(path, self.checkpoint_dir, filename)
return cp_path
def get_checkpoint_model(self, checkpoint_id, name, path=''):
"""construct the info dict for a given checkpoint"""
path = path.strip('/')
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
stats = os.stat(cp_path)
last_modified = tz.utcfromtimestamp(stats.st_mtime)
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
# public checkpoint API
def create_checkpoint(self, name, path=''):
"""Create a checkpoint from the current state of a notebook"""
path = path.strip('/')
nb_path = self.get_os_path(name, path)
# only the one checkpoint ID:
checkpoint_id = u"checkpoint"
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
self.log.debug("creating checkpoint for notebook %s", name)
if not os.path.exists(self.checkpoint_dir):
os.mkdir(self.checkpoint_dir)
shutil.copy2(nb_path, cp_path)
# return the checkpoint info
return self.get_checkpoint_model(checkpoint_id, name, path)
def list_checkpoints(self, name, path=''):
"""list the checkpoints for a given notebook
This notebook manager currently only supports one checkpoint per notebook.
"""
path = path.strip('/')
checkpoint_id = "checkpoint"
path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.exists(path):
return []
else:
return [self.get_checkpoint_model(checkpoint_id, name, path)]
def restore_checkpoint(self, checkpoint_id, name, path=''):
"""restore a notebook to a checkpointed state"""
path = path.strip('/')
self.log.info("restoring Notebook %s from checkpoint %s", name,
checkpoint_id)
nb_path = self.get_os_path(name, path)
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(
404, u'Notebook checkpoint does not exist: %s-%s' %
(name, checkpoint_id))
# ensure notebook is readable (never restore from an unreadable notebook)
with io.open(cp_path, 'r', encoding='utf-8') as f:
nb = current.read(f, u'json')
shutil.copy2(cp_path, nb_path)
self.log.debug("copying %s -> %s", cp_path, nb_path)
def delete_checkpoint(self, checkpoint_id, name, path=''):
"""delete a notebook's checkpoint"""
path = path.strip('/')
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.isfile(cp_path):
raise web.HTTPError(
404, u'Notebook checkpoint does not exist: %s%s-%s' %
(path, name, checkpoint_id))
self.log.debug("unlinking %s", cp_path)
os.unlink(cp_path)
def info_string(self):
return "Serving notebooks from local directory: %s" % self.notebook_dir
class Kernel(Configurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
def _eventloop_changed(self, name, old, new):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
shell_class = Type(ZMQInteractiveShell)
session = Instance(Session)
profile_dir = Instance('IPython.core.profiledir.ProfileDir')
shell_streams = List()
control_stream = Instance(ZMQStream)
iopub_socket = Instance(zmq.Socket)
stdin_socket = Instance(zmq.Socket)
log = Instance(logging.Logger)
user_module = Any()
def _user_module_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_module = new
user_ns = Instance(dict, args=None, allow_none=True)
def _user_ns_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_ns = new
self.shell.init_user_ns()
# identities:
int_id = Integer(-1)
ident = Unicode()
def _ident_default(self):
return unicode_type(uuid.uuid4())
# Private interface
_darwin_app_nap = Bool(
True,
config=True,
help="""Whether to use appnope for compatiblity with OS X App Nap.
Only affects OS X >= 10.9.
""")
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005, config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05, config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# A reference to the Python builtin 'raw_input' function.
# (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
_sys_raw_input = Any()
_sys_eval_input = Any()
# set of aborted msg_ids
aborted = Set()
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Initialize the InteractiveShell subclass
self.shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('pyout')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.shell.data_pub.session = self.session
self.shell.data_pub.pub_socket = self.iopub_socket
# TMP - hack while developing
self.shell._reply_content = None
# Build dict of handlers for message types
msg_types = [
'execute_request',
'complete_request',
'object_info_request',
'history_request',
'kernel_info_request',
'connect_request',
'shutdown_request',
'apply_request',
]
self.shell_handlers = {}
for msg_type in msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
comm_manager = self.shell.comm_manager
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(comm_manager, msg_type)
control_msg_types = msg_types + ['clear_request', 'abort_request']
self.control_handlers = {}
for msg_type in control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
header = msg['header']
msg_id = header['msg_id']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if msg_id in self.aborted:
self.aborted.remove(msg_id)
# is it safe to assume a msg_id will not be resubmitted?
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
# ensure default_int_handler during handler call
sig = signal(SIGINT, default_int_handler)
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
signal(SIGINT, sig)
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop %s", self.eventloop)
for stream in self.shell_streams:
# flush any pending replies,
# which may be skipped by entering the eventloop
stream.flush(zmq.POLLOUT)
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
self.shell.exit_now = False
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
# publish idle status
self._publish_status('starting')
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _make_metadata(self, other=None):
"""init metadata dict, for execute/apply_reply"""
new_md = {
'dependencies_met': True,
'engine': self.ident,
'started': datetime.now(),
}
if other:
new_md.update(other)
return new_md
def _publish_pyin(self, code, parent, execution_count):
"""Publish the code request on the pyin stream."""
self.session.send(self.iopub_socket,
u'pyin', {
u'code': code,
u'execution_count': execution_count
},
parent=parent,
ident=self._topic('pyin'))
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(
self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent,
ident=self._topic('status'),
)
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
self._publish_status(u'busy', parent)
try:
content = parent[u'content']
code = py3compat.cast_unicode_py2(content[u'code'])
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
md = self._make_metadata(parent['metadata'])
shell = self.shell # we'll need this a lot here
# Replace raw_input. Note that is not sufficient to replace
# raw_input in the user namespace.
if content.get('allow_stdin', False):
raw_input = lambda prompt='': self._raw_input(
prompt, ident, parent)
input = lambda prompt='': eval(raw_input(prompt))
else:
raw_input = input = lambda prompt='': self._no_raw_input()
if py3compat.PY3:
self._sys_raw_input = builtin_mod.input
builtin_mod.input = raw_input
else:
self._sys_raw_input = builtin_mod.raw_input
self._sys_eval_input = builtin_mod.input
builtin_mod.raw_input = raw_input
builtin_mod.input = input
# Set the parent message of the display hook and out streams.
shell.set_parent(parent)
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self._publish_pyin(code, parent, shell.execution_count)
reply_content = {}
try:
# FIXME: the shell calls the exception handler itself.
shell.run_cell(code, store_history=store_history, silent=silent)
except:
status = u'error'
# FIXME: this code right now isn't being used yet by default,
# because the run_cell() call above directly fires off exception
# reporting. This code, therefore, is only active in the scenario
# where runlines itself has an unhandled exception. We need to
# uniformize this, for all exception construction to come from a
# single location in the codbase.
etype, evalue, tb = sys.exc_info()
tb_list = traceback.format_exception(etype, evalue, tb)
reply_content.update(shell._showtraceback(etype, evalue, tb_list))
else:
status = u'ok'
finally:
# Restore raw_input.
if py3compat.PY3:
builtin_mod.input = self._sys_raw_input
else:
builtin_mod.raw_input = self._sys_raw_input
builtin_mod.input = self._sys_eval_input
reply_content[u'status'] = status
# Return the execution counter so clients can display prompts
reply_content['execution_count'] = shell.execution_count - 1
# FIXME - fish exception info out of shell, possibly left there by
# runlines. We'll need to clean up this logic later.
if shell._reply_content is not None:
reply_content.update(shell._reply_content)
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method='execute')
reply_content['engine_info'] = e_info
# reset after use
shell._reply_content = None
if 'traceback' in reply_content:
self.log.info("Exception in execute request:\n%s",
'\n'.join(reply_content['traceback']))
# At this point, we can tell whether the main code execution succeeded
# or not. If it did, we proceed to evaluate user_variables/expressions
if reply_content['status'] == 'ok':
reply_content[u'user_variables'] = \
shell.user_variables(content.get(u'user_variables', []))
reply_content[u'user_expressions'] = \
shell.user_expressions(content.get(u'user_expressions', {}))
else:
# If there was an error, don't even try to compute variables or
# expressions
reply_content[u'user_variables'] = {}
reply_content[u'user_expressions'] = {}
# Payloads should be retrieved regardless of outcome, so we can both
# recover partial output (that could have been generated early in a
# block, before an error) and clear the payload system always.
reply_content[u'payload'] = shell.payload_manager.read_payload()
# Be agressive about clearing the payload because we don't want
# it to sit in memory until the next execute_request comes in.
shell.payload_manager.clear_payload()
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
md['status'] = reply_content['status']
if reply_content['status'] == 'error' and \
reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent,
metadata=md,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content']['status'] == u'error':
self._abort_queues()
self._publish_status(u'idle', parent)
def complete_request(self, stream, ident, parent):
txt, matches = self._complete(parent)
matches = {'matches': matches, 'matched_text': txt, 'status': 'ok'}
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply', matches,
parent, ident)
self.log.debug("%s", completion_msg)
def object_info_request(self, stream, ident, parent):
content = parent['content']
object_info = self.shell.object_inspect(content['oname'],
detail_level=content.get(
'detail_level', 0))
# Before we send this object over, we scrub it for JSON usage
oinfo = json_clean(object_info)
msg = self.session.send(stream, 'object_info_reply', oinfo, parent,
ident)
self.log.debug("%s", msg)
def history_request(self, stream, ident, parent):
# We need to pull these out, as passing **kwargs doesn't work with
# unicode keys before Python 2.6.5.
hist_access_type = parent['content']['hist_access_type']
raw = parent['content']['raw']
output = parent['content']['output']
if hist_access_type == 'tail':
n = parent['content']['n']
hist = self.shell.history_manager.get_tail(n,
raw=raw,
output=output,
include_latest=True)
elif hist_access_type == 'range':
session = parent['content']['session']
start = parent['content']['start']
stop = parent['content']['stop']
hist = self.shell.history_manager.get_range(session,
start,
stop,
raw=raw,
output=output)
elif hist_access_type == 'search':
n = parent['content'].get('n')
unique = parent['content'].get('unique', False)
pattern = parent['content']['pattern']
hist = self.shell.history_manager.search(pattern,
raw=raw,
output=output,
n=n,
unique=unique)
else:
hist = []
hist = list(hist)
content = {'history': hist}
content = json_clean(content)
msg = self.session.send(stream, 'history_reply', content, parent,
ident)
self.log.debug("Sending history reply with %i entries", len(hist))
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
msg = self.session.send(stream, 'connect_reply', content, parent,
ident)
self.log.debug("%s", msg)
def kernel_info_request(self, stream, ident, parent):
vinfo = {
'protocol_version': protocol_version,
'ipython_version': ipython_version,
'language_version': language_version,
'language': 'python',
}
msg = self.session.send(stream, 'kernel_info_reply', vinfo, parent,
ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
self.shell.exit_now = True
content = dict(status='ok')
content.update(parent['content'])
self.session.send(stream,
u'shutdown_reply',
content,
parent,
ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply', content,
parent)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time() + 0.1, loop.stop)
#---------------------------------------------------------------------------
# Engine methods
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
self._publish_status(u'busy', parent)
# Set the parent message of the display hook and out streams.
shell = self.shell
shell.set_parent(parent)
# pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
# self.iopub_socket.send(pyin_msg)
# self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
md = self._make_metadata(parent['metadata'])
try:
working = shell.user_ns
prefix = "_" + str(msg_id).replace("-", "") + "_"
f, args, kwargs = unpack_apply_message(bufs, working, copy=False)
fname = getattr(f, '__name__', 'f')
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
ns = {fname: f, argname: args, kwargname: kwargs, resultname: None}
# print ns
working.update(ns)
code = "%s = %s(*%s,**%s)" % (resultname, fname, argname,
kwargname)
try:
exec(code, shell.user_global_ns, shell.user_ns)
result = working.get(resultname)
finally:
for key in ns:
working.pop(key)
result_buf = serialize_object(
result,
buffer_threshold=self.session.buffer_threshold,
item_threshold=self.session.item_threshold,
)
except:
# invoke IPython traceback formatting
shell.showtraceback()
# FIXME - fish exception info out of shell, possibly left there by
# run_code. We'll need to clean up this logic later.
reply_content = {}
if shell._reply_content is not None:
reply_content.update(shell._reply_content)
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method='apply')
reply_content['engine_info'] = e_info
# reset after use
shell._reply_content = None
self.session.send(self.iopub_socket,
u'pyerr',
reply_content,
parent=parent,
ident=self._topic('pyerr'))
self.log.info("Exception in apply request:\n%s",
'\n'.join(reply_content['traceback']))
result_buf = []
if reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
else:
reply_content = {'status': 'ok'}
# put 'ok'/'error' status in header, for scheduler introspection:
md['status'] = reply_content['status']
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
reply_msg = self.session.send(stream,
u'apply_reply',
reply_content,
parent=parent,
ident=ident,
buffers=result_buf,
metadata=md)
self._publish_status(u'idle', parent)
#---------------------------------------------------------------------------
# Control messages
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specifig msg by id"""
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, string_types):
msg_ids = [msg_ids]
if not msg_ids:
self.abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream,
'abort_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
self.shell.reset(False)
msg = self.session.send(stream,
'clear_reply',
ident=idents,
parent=parent,
content=dict(status='ok'))
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _wrap_exception(self, method=None):
# import here, because _wrap_exception is only used in parallel,
# and parallel has higher min pyzmq version
from IPython.parallel.error import wrap_exception
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method=method)
content = wrap_exception(e_info)
return content
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
if self.int_id >= 0:
base = "engine.%i" % self.int_id
else:
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def _raw_input(self, prompt, ident, parent):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# flush the stdin socket, to purge stale replies
while True:
try:
self.stdin_socket.recv_multipart(zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
break
else:
raise
# Send the input request.
content = json_clean(dict(prompt=prompt))
self.session.send(self.stdin_socket,
u'input_request',
content,
parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
except KeyboardInterrupt:
# re-raise KeyboardInterrupt, to truncate traceback
raise KeyboardInterrupt
else:
break
try:
value = py3compat.unicode_to_str(reply['content']['value'])
except:
self.log.error("Got bad raw_input reply: ")
self.log.error("%s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _complete(self, msg):
c = msg['content']
try:
cpos = int(c['cursor_pos'])
except:
# If we don't get something that we can convert to an integer, at
# least attempt the completion guessing the cursor is at the end of
# the text, if there's any, and otherwise of the line
cpos = len(c['text'])
if cpos == 0:
cpos = len(c['line'])
return self.shell.complete(c['text'], c['line'], cpos)
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket,
self._shutdown_message,
ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[s.flush(zmq.POLLOUT) for s in self.shell_streams]
class UnicodeTrait(HasTraits):
value = Unicode(u'unicode')
class Exporter(Configurable):
"""
Exports notebooks into other file formats. Uses Jinja 2 templating engine
to output new formats. Inherit from this class if you are creating a new
template type along with new filters/transformers. If the filters/
transformers provided by default suffice, there is no need to inherit from
this class. Instead, override the template_file and file_extension
traits via a config file.
{filters}
"""
# finish the docstring
__doc__ = __doc__.format(filters='- ' +
'\n - '.join(default_filters.keys()))
template_file = Unicode('',
config=True,
help="Name of the template file to use")
file_extension = Unicode(
'txt',
config=True,
help="Extension of the file that should be written to disk")
template_path = Unicode("/../templates/",
config=True,
help="Path where the template files are located.")
template_skeleton_path = Unicode(
"/../templates/skeleton/",
config=True,
help="Path where the template skeleton files are located.")
#Jinja block definitions
jinja_comment_block_start = Unicode("", config=True)
jinja_comment_block_end = Unicode("", config=True)
jinja_variable_block_start = Unicode("", config=True)
jinja_variable_block_end = Unicode("", config=True)
jinja_logic_block_start = Unicode("", config=True)
jinja_logic_block_end = Unicode("", config=True)
#Extension that the template files use.
template_extension = Unicode(".tpl", config=True)
#Processors that process the input data prior to the export, set in the
#constructor for this class.
transformers = None
_default_config = Config()
def __init__(self, transformers=None, filters=None, config=None, **kw):
"""
Public constructor
Parameters
----------
transformers : list[of transformer]
Custom transformers to apply to the notebook prior to engaging
the Jinja template engine. Any transformers specified here
will override existing transformers if a naming conflict
occurs.
filters : dict[of filter]
filters specified here will override existing filters if a naming
conflict occurs. Filters are availlable in jinja template through
the name of the corresponding key. Cf class docstring for
availlable default filters.
config : config
User configuration instance.
"""
#Call the base class constructor
super(Exporter, self).__init__(config=config, **kw)
#Standard environment
self._init_environment()
#Add transformers
self._register_transformers()
#Add filters to the Jinja2 environment
self._register_filters()
#Load user transformers. Overwrite existing transformers if need be.
if transformers:
for transformer in transformers:
self.register_transformer(transformer)
#Load user filters. Overwrite existing filters if need be.
if not filters is None:
for key, user_filter in filters.iteritems():
if issubclass(user_filter, MetaHasTraits):
self.environment.filters[key] = user_filter(config=config)
else:
self.environment.filters[key] = user_filter
@property
def default_config(self):
if self._default_config:
return Config(deepcopy(self._default_config))
else:
return Config()
def from_notebook_node(self, nb, resources=None):
"""
Convert a notebook from a notebook node instance.
Parameters
----------
nb : Notebook node
resources : a dict of additional resources that
can be accessed read/write by transformers
and filters.
"""
if resources is None:
resources = {}
nb, resources = self._preprocess(nb, resources)
#Load the template file.
self.template = self.environment.get_template(self.template_file +
self.template_extension)
return self.template.render(nb=nb, resources=resources), resources
def from_filename(self, filename):
"""
Convert a notebook from a notebook file.
Parameters
----------
filename : str
Full filename of the notebook file to open and convert.
"""
with io.open(filename) as f:
return self.from_notebook_node(nbformat.read(f, 'json'))
def from_file(self, file_stream):
"""
Convert a notebook from a notebook file.
Parameters
----------
file_stream : file-like object
Notebook file-like object to convert.
"""
return self.from_notebook_node(nbformat.read(file_stream, 'json'))
def register_transformer(self, transformer):
"""
Register a transformer.
Transformers are classes that act upon the notebook before it is
passed into the Jinja templating engine. Transformers are also
capable of passing additional information to the Jinja
templating engine.
Parameters
----------
transformer : transformer
"""
if self.transformers is None:
self.transformers = []
if inspect.isfunction(transformer):
self.transformers.append(transformer)
return transformer
elif isinstance(transformer, MetaHasTraits):
transformer_instance = transformer(config=self.config)
self.transformers.append(transformer_instance)
return transformer_instance
else:
transformer_instance = transformer()
self.transformers.append(transformer_instance)
return transformer_instance
def register_filter(self, name, filter):
"""
Register a filter.
A filter is a function that accepts and acts on one string.
The filters are accesible within the Jinja templating engine.
Parameters
----------
name : str
name to give the filter in the Jinja engine
filter : filter
"""
if inspect.isfunction(filter):
self.environment.filters[name] = filter
elif isinstance(filter, MetaHasTraits):
self.environment.filters[name] = filter(config=self.config)
else:
self.environment.filters[name] = filter()
return self.environment.filters[name]
def _register_transformers(self):
"""
Register all of the transformers needed for this exporter.
"""
self.register_transformer(music21.ext.nbconvert.transformers.
coalescestreams.coalesce_streams)
#Remember the figure extraction transformer so it can be enabled and
#disabled easily later.
self.extract_figure_transformer = self.register_transformer(
music21.ext.nbconvert.transformers.extractfigure.
ExtractFigureTransformer)
def _register_filters(self):
"""
Register all of the filters required for the exporter.
"""
for k, v in default_filters.iteritems():
self.register_filter(k, v)
def _init_environment(self):
"""
Create the Jinja templating environment.
"""
self.environment = Environment(loader=FileSystemLoader([
os.path.dirname(os.path.realpath(__file__)) + self.template_path,
os.path.dirname(os.path.realpath(__file__)) +
self.template_skeleton_path,
]),
extensions=JINJA_EXTENSIONS)
#Set special Jinja2 syntax that will not conflict with latex.
if self.jinja_logic_block_start:
self.environment.block_start_string = self.jinja_logic_block_start
if self.jinja_logic_block_end:
self.environment.block_end_string = self.jinja_logic_block_end
if self.jinja_variable_block_start:
self.environment.variable_start_string = self.jinja_variable_block_start
if self.jinja_variable_block_end:
self.environment.variable_end_string = self.jinja_variable_block_end
if self.jinja_comment_block_start:
self.environment.comment_start_string = self.jinja_comment_block_start
if self.jinja_comment_block_end:
self.environment.comment_end_string = self.jinja_comment_block_end
def _preprocess(self, nb, resources):
"""
Preprocess the notebook before passing it into the Jinja engine.
To preprocess the notebook is to apply all of the
Parameters
----------
nb : notebook node
notebook that is being exported.
resources : a dict of additional resources that
can be accessed read/write by transformers
and filters.
"""
# Do a deepcopy first,
# we are never safe enough with what the transformers could do.
nbc = deepcopy(nb)
resc = deepcopy(resources)
#Run each transformer on the notebook. Carry the output along
#to each transformer
for transformer in self.transformers:
nb, resources = transformer(nbc, resc)
return nb, resources
class IntProgressWidget(_BoundedIntWidget):
_view_name = Unicode('ProgressView', sync=True)
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):
""" A Qt frontend for a generic Python kernel.
"""
# The text to show when the kernel is (re)started.
banner = Unicode(config=True)
# An option and corresponding signal for overriding the default kernel
# interrupt behavior.
custom_interrupt = Bool(False)
custom_interrupt_requested = QtCore.Signal()
# An option and corresponding signals for overriding the default kernel
# restart behavior.
custom_restart = Bool(False)
custom_restart_kernel_died = QtCore.Signal(float)
custom_restart_requested = QtCore.Signal()
# Whether to automatically show calltips on open-parentheses.
enable_calltips = Bool(
True,
config=True,
help="Whether to draw information calltips on open-parentheses.")
clear_on_kernel_restart = Bool(
True,
config=True,
help="Whether to clear the console when the kernel is restarted")
confirm_restart = Bool(
True,
config=True,
help="Whether to ask for user confirmation when restarting kernel")
# Emitted when a user visible 'execute_request' has been submitted to the
# kernel from the FrontendWidget. Contains the code to be executed.
executing = QtCore.Signal(object)
# Emitted when a user-visible 'execute_reply' has been received from the
# kernel and processed by the FrontendWidget. Contains the response message.
executed = QtCore.Signal(object)
# Emitted when an exit request has been received from the kernel.
exit_requested = QtCore.Signal(object)
# Protected class variables.
_prompt_transformer = IPythonInputSplitter(
physical_line_transforms=[classic_prompt()],
logical_line_transforms=[],
python_line_transforms=[],
)
_CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos'])
_CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos'])
_ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
_input_splitter_class = InputSplitter
_local_kernel = False
_highlighter = Instance(FrontendHighlighter)
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(FrontendWidget, self).__init__(*args, **kw)
# FIXME: remove this when PySide min version is updated past 1.0.7
# forcefully disable calltips if PySide is < 1.0.7, because they crash
if qt.QT_API == qt.QT_API_PYSIDE:
import PySide
if PySide.__version_info__ < (1, 0, 7):
self.log.warn(
"PySide %s < 1.0.7 detected, disabling calltips" %
PySide.__version__)
self.enable_calltips = False
# FrontendWidget protected variables.
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._completion_lexer = CompletionLexer(PythonLexer())
self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
self._hidden = False
self._highlighter = FrontendHighlighter(self)
self._input_splitter = self._input_splitter_class()
self._kernel_manager = None
self._kernel_client = None
self._request_info = {}
self._request_info['execute'] = {}
self._callback_dict = {}
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Configure actions.
action = self._copy_raw_action
key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C
action.setEnabled(False)
action.setShortcut(QtGui.QKeySequence(key))
action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut)
action.triggered.connect(self.copy_raw)
self.copy_available.connect(action.setEnabled)
self.addAction(action)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Set flag for whether we are connected via localhost.
self._local_kernel = kw.get('local_kernel',
FrontendWidget._local_kernel)
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
def copy(self):
""" Copy the currently selected text to the clipboard, removing prompts.
"""
if self._page_control is not None and self._page_control.hasFocus():
self._page_control.copy()
elif self._control.hasFocus():
text = self._control.textCursor().selection().toPlainText()
if text:
text = self._prompt_transformer.transform_cell(text)
QtGui.QApplication.clipboard().setText(text)
else:
self.log.debug("frontend widget : unknown copy target")
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _is_complete(self, source, interactive):
""" Returns whether 'source' can be completely processed and a new
prompt created. When triggered by an Enter/Return key press,
'interactive' is True; otherwise, it is False.
"""
self._input_splitter.reset()
complete = self._input_splitter.push(source)
if interactive:
complete = not self._input_splitter.push_accepts_more()
return complete
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
msg_id = self.kernel_client.execute(source, hidden)
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'user')
self._hidden = hidden
if not hidden:
self.executing.emit(source)
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
# Flush all state from the input splitter so the next round of
# reading input starts with a clean buffer.
self._input_splitter.reset()
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _context_menu_make(self, pos):
""" Reimplemented to add an action for raw copy.
"""
menu = super(FrontendWidget, self)._context_menu_make(pos)
for before_action in menu.actions():
if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \
QtGui.QKeySequence.ExactMatch:
menu.insertAction(before_action, self._copy_raw_action)
break
return menu
def request_interrupt_kernel(self):
if self._executing:
self.interrupt_kernel()
def request_restart_kernel(self):
message = 'Are you sure you want to restart the kernel?'
self.restart_kernel(message, now=False)
def _event_filter_console_keypress(self, event):
""" Reimplemented for execution interruption and smart backspace.
"""
key = event.key()
if self._control_key_down(event.modifiers(), include_command=False):
if key == QtCore.Qt.Key_C and self._executing:
self.request_interrupt_kernel()
return True
elif key == QtCore.Qt.Key_Period:
self.request_restart_kernel()
return True
elif not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
if key == QtCore.Qt.Key_Backspace:
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(FrontendWidget,
self)._event_filter_console_keypress(event)
def _insert_continuation_prompt(self, cursor):
""" Reimplemented for auto-indentation.
"""
super(FrontendWidget, self)._insert_continuation_prompt(cursor)
cursor.insertText(' ' * self._input_splitter.indent_spaces)
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_complete_reply(self, rep):
""" Handle replies for tab completion.
"""
self.log.debug("complete: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('complete')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
text = '.'.join(self._get_context())
cursor.movePosition(QtGui.QTextCursor.Left, n=len(text))
self._complete_with_items(cursor, rep['content']['matches'])
def _silent_exec_callback(self, expr, callback):
"""Silently execute `expr` in the kernel and call `callback` with reply
the `expr` is evaluated silently in the kernel (without) output in
the frontend. Call `callback` with the
`repr <http://docs.python.org/library/functions.html#repr> `_ as first argument
Parameters
----------
expr : string
valid string to be executed by the kernel.
callback : function
function accepting one argument, as a string. The string will be
the `repr` of the result of evaluating `expr`
The `callback` is called with the `repr()` of the result of `expr` as
first argument. To get the object, do `eval()` on the passed value.
See Also
--------
_handle_exec_callback : private method, deal with calling callback with reply
"""
# generate uuid, which would be used as an indication of whether or
# not the unique request originated from here (can use msg id ?)
local_uuid = str(uuid.uuid1())
msg_id = self.kernel_client.execute(
'', silent=True, user_expressions={local_uuid: expr})
self._callback_dict[local_uuid] = callback
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'silent_exec_callback')
def _handle_exec_callback(self, msg):
"""Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``
Parameters
----------
msg : raw message send by the kernel containing an `user_expressions`
and having a 'silent_exec_callback' kind.
Notes
-----
This function will look for a `callback` associated with the
corresponding message id. Association has been made by
`_silent_exec_callback`. `callback` is then called with the `repr()`
of the value of corresponding `user_expressions` as argument.
`callback` is then removed from the known list so that any message
coming again with the same id won't trigger it.
"""
user_exp = msg['content'].get('user_expressions')
if not user_exp:
return
for expression in user_exp:
if expression in self._callback_dict:
self._callback_dict.pop(expression)(user_exp[expression])
def _handle_execute_reply(self, msg):
""" Handles replies for code execution.
"""
self.log.debug("execute: %s", msg.get('content', ''))
msg_id = msg['parent_header']['msg_id']
info = self._request_info['execute'].get(msg_id)
# unset reading flag, because if execute finished, raw_input can't
# still be pending.
self._reading = False
if info and info.kind == 'user' and not self._hidden:
# Make sure that all output from the SUB channel has been processed
# before writing a new prompt.
self.kernel_client.iopub_channel.flush()
# Reset the ANSI style information to prevent bad text in stdout
# from messing up our colors. We're not a true terminal so we're
# allowed to do this.
if self.ansi_codes:
self._ansi_processor.reset_sgr()
content = msg['content']
status = content['status']
if status == 'ok':
self._process_execute_ok(msg)
elif status == 'error':
self._process_execute_error(msg)
elif status == 'aborted':
self._process_execute_abort(msg)
self._show_interpreter_prompt_for_reply(msg)
self.executed.emit(msg)
self._request_info['execute'].pop(msg_id)
elif info and info.kind == 'silent_exec_callback' and not self._hidden:
self._handle_exec_callback(msg)
self._request_info['execute'].pop(msg_id)
else:
super(FrontendWidget, self)._handle_execute_reply(msg)
def _handle_input_request(self, msg):
""" Handle requests for raw_input.
"""
self.log.debug("input: %s", msg.get('content', ''))
if self._hidden:
raise RuntimeError(
'Request for raw input during hidden execution.')
# Make sure that all output from the SUB channel has been processed
# before entering readline mode.
self.kernel_client.iopub_channel.flush()
def callback(line):
self.kernel_client.stdin_channel.input(line)
if self._reading:
self.log.debug(
"Got second input request, assuming first was interrupted.")
self._reading = False
self._readline(msg['content']['prompt'], callback=callback)
def _kernel_restarted_message(self, died=True):
msg = "Kernel died, restarting" if died else "Kernel restarting"
self._append_html("<br>%s<hr><br>" % msg, before_prompt=False)
def _handle_kernel_died(self, since_last_heartbeat):
"""Handle the kernel's death (if we do not own the kernel).
"""
self.log.warn("kernel died: %s", since_last_heartbeat)
if self.custom_restart:
self.custom_restart_kernel_died.emit(since_last_heartbeat)
else:
self._kernel_restarted_message(died=True)
self.reset()
def _handle_kernel_restarted(self, died=True):
"""Notice that the autorestarter restarted the kernel.
There's nothing to do but show a message.
"""
self.log.warn("kernel restarted")
self._kernel_restarted_message(died=died)
self.reset()
def _handle_object_info_reply(self, rep):
""" Handle replies for call tips.
"""
self.log.debug("oinfo: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('call_tip')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
# Get the information for a call tip. For now we format the call
# line as string, later we can pass False to format_call and
# syntax-highlight it ourselves for nicer formatting in the
# calltip.
content = rep['content']
# if this is from pykernel, 'docstring' will be the only key
if content.get('ismagic', False):
# Don't generate a call-tip for magics. Ideally, we should
# generate a tooltip, but not on ( like we do for actual
# callables.
call_info, doc = None, None
else:
call_info, doc = call_tip(content, format_call=True)
if call_info or doc:
self._call_tip_widget.show_call_info(call_info, doc)
def _handle_pyout(self, msg):
""" Handle display hook output.
"""
self.log.debug("pyout: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
text = msg['content']['data']
self._append_plain_text(text + '\n', before_prompt=True)
def _handle_stream(self, msg):
""" Handle stdout, stderr, and stdin.
"""
self.log.debug("stream: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
# Most consoles treat tabs as being 8 space characters. Convert tabs
# to spaces so that output looks as expected regardless of this
# widget's tab width.
text = msg['content']['data'].expandtabs(8)
self._append_plain_text(text, before_prompt=True)
self._control.moveCursor(QtGui.QTextCursor.End)
def _handle_shutdown_reply(self, msg):
""" Handle shutdown signal, only if from other console.
"""
self.log.warn("shutdown: %s", msg.get('content', ''))
restart = msg.get('content', {}).get('restart', False)
if not self._hidden and not self._is_from_this_session(msg):
# got shutdown reply, request came from session other than ours
if restart:
# someone restarted the kernel, handle it
self._handle_kernel_restarted(died=False)
else:
# kernel was shutdown permanently
# this triggers exit_requested if the kernel was local,
# and a dialog if the kernel was remote,
# so we don't suddenly clear the qtconsole without asking.
if self._local_kernel:
self.exit_requested.emit(self)
else:
title = self.window().windowTitle()
reply = QtGui.QMessageBox.question(
self, title, "Kernel has been shutdown permanently. "
"Close the Console?", QtGui.QMessageBox.Yes,
QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
self.exit_requested.emit(self)
def _handle_status(self, msg):
"""Handle status message"""
# This is where a busy/idle indicator would be triggered,
# when we make one.
state = msg['content'].get('execution_state', '')
if state == 'starting':
# kernel started while we were running
if self._executing:
self._handle_kernel_restarted(died=True)
elif state == 'idle':
pass
elif state == 'busy':
pass
def _started_channels(self):
""" Called when the KernelManager channels have started listening or
when the frontend is assigned an already listening KernelManager.
"""
self.reset(clear=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def copy_raw(self):
""" Copy the currently selected text to the clipboard without attempting
to remove prompts or otherwise alter the text.
"""
self._control.copy()
def execute_file(self, path, hidden=False):
""" Attempts to execute file with 'path'. If 'hidden', no output is
shown.
"""
self.execute('execfile(%r)' % path, hidden=hidden)
def interrupt_kernel(self):
""" Attempts to interrupt the running kernel.
Also unsets _reading flag, to avoid runtime errors
if raw_input is called again.
"""
if self.custom_interrupt:
self._reading = False
self.custom_interrupt_requested.emit()
elif self.kernel_manager:
self._reading = False
self.kernel_manager.interrupt_kernel()
else:
self._append_plain_text(
'Cannot interrupt a kernel I did not start.\n')
def reset(self, clear=False):
""" Resets the widget to its initial state if ``clear`` parameter
is True, otherwise
prints a visual indication of the fact that the kernel restarted, but
does not clear the traces from previous usage of the kernel before it
was restarted. With ``clear=True``, it is similar to ``%clear``, but
also re-writes the banner and aborts execution if necessary.
"""
if self._executing:
self._executing = False
self._request_info['execute'] = {}
self._reading = False
self._highlighter.highlighting_on = False
if clear:
self._control.clear()
self._append_plain_text(self.banner)
# update output marker for stdout/stderr, so that startup
# messages appear after banner:
self._append_before_prompt_pos = self._get_cursor().position()
self._show_interpreter_prompt()
def restart_kernel(self, message, now=False):
""" Attempts to restart the running kernel.
"""
# FIXME: now should be configurable via a checkbox in the dialog. Right
# now at least the heartbeat path sets it to True and the manual restart
# to False. But those should just be the pre-selected states of a
# checkbox that the user could override if so desired. But I don't know
# enough Qt to go implementing the checkbox now.
if self.custom_restart:
self.custom_restart_requested.emit()
return
if self.kernel_manager:
# Pause the heart beat channel to prevent further warnings.
self.kernel_client.hb_channel.pause()
# Prompt the user to restart the kernel. Un-pause the heartbeat if
# they decline. (If they accept, the heartbeat will be un-paused
# automatically when the kernel is restarted.)
if self.confirm_restart:
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
result = QtGui.QMessageBox.question(self, 'Restart kernel?',
message, buttons)
do_restart = result == QtGui.QMessageBox.Yes
else:
# confirm_restart is False, so we don't need to ask user
# anything, just do the restart
do_restart = True
if do_restart:
try:
self.kernel_manager.restart_kernel(now=now)
except RuntimeError as e:
self._append_plain_text('Error restarting kernel: %s\n' %
e,
before_prompt=True)
else:
self._append_html(
"<br>Restarting kernel...\n<hr><br>",
before_prompt=True,
)
else:
self.kernel_client.hb_channel.unpause()
else:
self._append_plain_text(
'Cannot restart a Kernel I did not start\n',
before_prompt=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _call_tip(self):
""" Shows a call tip, if appropriate, at the current cursor location.
"""
# Decide if it makes sense to show a call tip
if not self.enable_calltips:
return False
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) != '(':
return False
context = self._get_context(cursor)
if not context:
return False
# Send the metadata request to the kernel
name = '.'.join(context)
msg_id = self.kernel_client.object_info(name)
pos = self._get_cursor().position()
self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
return True
def _complete(self):
""" Performs completion at the current cursor location.
"""
context = self._get_context()
if context:
# Send the completion request to the kernel
msg_id = self.kernel_client.complete(
'.'.join(context), # text
self._get_input_buffer_cursor_line(), # line
self._get_input_buffer_cursor_column(), # cursor_pos
self.input_buffer) # block
pos = self._get_cursor().position()
info = self._CompletionRequest(msg_id, pos)
self._request_info['complete'] = info
def _get_context(self, cursor=None):
""" Gets the context for the specified cursor (or the current cursor
if none is specified).
"""
if cursor is None:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.StartOfBlock,
QtGui.QTextCursor.KeepAnchor)
text = cursor.selection().toPlainText()
return self._completion_lexer.get_context(text)
def _process_execute_abort(self, msg):
""" Process a reply for an aborted execution request.
"""
self._append_plain_text("ERROR: execution aborted\n")
def _process_execute_error(self, msg):
""" Process a reply for an execution request that resulted in an error.
"""
content = msg['content']
# If a SystemExit is passed along, this means exit() was called - also
# all the ipython %exit magic syntax of '-k' to be used to keep
# the kernel running
if content['ename'] == 'SystemExit':
keepkernel = content['evalue'] == '-k' or content[
'evalue'] == 'True'
self._keep_kernel_on_exit = keepkernel
self.exit_requested.emit(self)
else:
traceback = ''.join(content['traceback'])
self._append_plain_text(traceback)
def _process_execute_ok(self, msg):
""" Process a reply for a successful execution request.
"""
payload = msg['content']['payload']
for item in payload:
if not self._process_execute_payload(item):
warning = 'Warning: received unknown payload of type %s'
print(warning % repr(item['source']))
def _process_execute_payload(self, item):
""" Process a single payload item from the list of payload items in an
execution reply. Returns whether the payload was handled.
"""
# The basic FrontendWidget doesn't handle payloads, as they are a
# mechanism for going beyond the standard Python interpreter model.
return False
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self._show_prompt('>>> ')
def _show_interpreter_prompt_for_reply(self, msg):
""" Shows a prompt for the interpreter given an 'execute_reply' message.
"""
self._show_interpreter_prompt()
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._call_tip()
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
""" Returns the standard Python banner.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
class Foo(Configurable):
i = Integer(0, config=True, help="The integer i.")
j = Integer(1, config=True, help="The integer j.")
name = Unicode(u'Brian', config=True, help="First name.")
class ProfileList(Application):
name = 'ipython-profile'
description = list_help
examples = _list_examples
aliases = Dict({
'ipython-dir': 'ProfileList.ipython_dir',
'log-level': 'Application.log_level',
})
flags = Dict(
dict(debug=({
'Application': {
'log_level': 0
}
}, "Set Application.log_level to 0, maximizing log output.")))
ipython_dir = Unicode(get_ipython_dir(),
config=True,
help="""
The name of the IPython directory. This directory is used for logging
configuration (through profiles), history storage, etc. The default
is usually $HOME/.ipython. This options can also be specified through
the environment variable IPYTHONDIR.
""")
def _print_profiles(self, profiles):
"""print list of profiles, indented."""
for profile in profiles:
print(' %s' % profile)
def list_profile_dirs(self):
profiles = list_bundled_profiles()
if profiles:
print()
print("Available profiles in IPython:")
self._print_profiles(profiles)
print()
print(" The first request for a bundled profile will copy it")
print(" into your IPython directory (%s)," % self.ipython_dir)
print(" where you can customize it.")
profiles = list_profiles_in(self.ipython_dir)
if profiles:
print()
print("Available profiles in %s:" % self.ipython_dir)
self._print_profiles(profiles)
profiles = list_profiles_in(os.getcwd())
if profiles:
print()
print("Available profiles in current directory (%s):" %
os.getcwd())
self._print_profiles(profiles)
print()
print("To use any of the above profiles, start IPython with:")
print(" ipython --profile=<name>")
print()
def start(self):
self.list_profile_dirs()
class Comm(LoggingConfigurable):
# If this is instantiated by a non-IPython kernel, shell will be None
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
kernel = Instance('IPython.kernel.zmq.kernelbase.Kernel')
def _kernel_default(self):
if Kernel.initialized():
return Kernel.instance()
iopub_socket = Any()
def _iopub_socket_default(self):
return self.kernel.iopub_socket
session = Instance('IPython.kernel.zmq.session.Session')
def _session_default(self):
if self.kernel is not None:
return self.kernel.session
target_name = Unicode('comm')
topic = Bytes()
def _topic_default(self):
return ('comm-%s' % self.comm_id).encode('ascii')
_open_data = Dict(help="data dict, if any, to be included in comm_open")
_close_data = Dict(help="data dict, if any, to be included in comm_close")
_msg_callback = Any()
_close_callback = Any()
_closed = Bool(False)
comm_id = Unicode()
def _comm_id_default(self):
return uuid.uuid4().hex
primary = Bool(True, help="Am I the primary or secondary Comm?")
def __init__(self, target_name='', data=None, **kwargs):
if target_name:
kwargs['target_name'] = target_name
super(Comm, self).__init__(**kwargs)
if self.primary:
# I am primary, open my peer.
self.open(data)
def _publish_msg(self,
msg_type,
data=None,
metadata=None,
buffers=None,
**keys):
"""Helper for sending a comm message on IOPub"""
data = {} if data is None else data
metadata = {} if metadata is None else metadata
content = json_clean(dict(data=data, comm_id=self.comm_id, **keys))
self.session.send(
self.iopub_socket,
msg_type,
content,
metadata=json_clean(metadata),
parent=self.kernel._parent_header,
ident=self.topic,
buffers=buffers,
)
def __del__(self):
"""trigger close on gc"""
self.close()
# publishing messages
def open(self, data=None, metadata=None, buffers=None):
"""Open the frontend-side version of this comm"""
if data is None:
data = self._open_data
comm_manager = getattr(self.kernel, 'comm_manager', None)
if comm_manager is None:
raise RuntimeError("Comms cannot be opened without a kernel "
"and a comm_manager attached to that kernel.")
comm_manager.register_comm(self)
self._closed = False
self._publish_msg(
'comm_open',
data=data,
metadata=metadata,
buffers=buffers,
target_name=self.target_name,
)
def close(self, data=None, metadata=None, buffers=None):
"""Close the frontend-side version of this comm"""
if self._closed:
# only close once
return
if data is None:
data = self._close_data
self._publish_msg(
'comm_close',
data=data,
metadata=metadata,
buffers=buffers,
)
self.kernel.comm_manager.unregister_comm(self)
self._closed = True
def send(self, data=None, metadata=None, buffers=None):
"""Send a message to the frontend-side version of this comm"""
self._publish_msg(
'comm_msg',
data=data,
metadata=metadata,
buffers=buffers,
)
# registering callbacks
def on_close(self, callback):
"""Register a callback for comm_close
Will be called with the `data` of the close message.
Call `on_close(None)` to disable an existing callback.
"""
self._close_callback = callback
def on_msg(self, callback):
"""Register a callback for comm_msg
Will be called with the `data` of any comm_msg messages.
Call `on_msg(None)` to disable an existing callback.
"""
self._msg_callback = callback
# handling of incoming messages
def handle_close(self, msg):
"""Handle a comm_close message"""
self.log.debug("handle_close[%s](%s)", self.comm_id, msg)
if self._close_callback:
self._close_callback(msg)
def handle_msg(self, msg):
"""Handle a comm_msg message"""
self.log.debug("handle_msg[%s](%s)", self.comm_id, msg)
if self._msg_callback:
if self.shell:
self.shell.events.trigger('pre_execute')
self._msg_callback(msg)
if self.shell:
self.shell.events.trigger('post_execute')
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
aliases = aliases
flags = flags
classes = [
KernelManager,
ProfileDir,
Session,
MappingKernelManager,
ContentsManager,
FileContentsManager,
NotebookNotary,
]
flags = Dict(flags)
aliases = Dict(aliases)
subcommands = dict(list=(NbserverListApp,
NbserverListApp.description.splitlines()[0]), )
ipython_kernel_argv = List(Unicode)
_log_formatter_cls = LogFormatter
def _log_level_default(self):
return logging.INFO
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%H:%M:%S"
def _log_format_default(self):
"""override default log format to include time"""
return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s"
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('', config=True)
# Network related information
allow_origin = Unicode('',
config=True,
help="""Set the Access-Control-Allow-Origin header
Use '*' to allow any origin to access your server.
Takes precedence over allow_origin_pat.
""")
allow_origin_pat = Unicode(
'',
config=True,
help=
"""Use a regular expression for the Access-Control-Allow-Origin header
Requests from an origin matching the expression will get replies with:
Access-Control-Allow-Origin: origin
where `origin` is the origin of the request.
Ignored if allow_origin is set.
""")
allow_credentials = Bool(
False,
config=True,
help="Set the Access-Control-Allow-Credentials: true header")
default_url = Unicode('/tree',
config=True,
help="The default URL to redirect to from `/`")
ip = Unicode('localhost',
config=True,
help="The IP address the notebook server will listen on.")
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888,
config=True,
help="The port the notebook server will listen on.")
port_retries = Integer(
50,
config=True,
help=
"The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(
u'',
config=True,
help="""The full path to an SSL/TLS certificate file.""")
keyfile = Unicode(
u'',
config=True,
help="""The full path to a private key file for usage with SSL/TLS.""")
cookie_secret_file = Unicode(
config=True, help="""The file where the cookie secret is stored.""")
def _cookie_secret_file_default(self):
if self.profile_dir is None:
return ''
return os.path.join(self.profile_dir.security_dir,
'notebook_cookie_secret')
cookie_secret = Bytes(b'',
config=True,
help="""The random bytes used to secure cookies.
By default this is a new random number every time you start the Notebook.
Set it to a value in a config file to enable logins to persist across server sessions.
Note: Cookie secrets should be kept private, do not share config files with
cookie_secret stored in plaintext (you can read the value from a file).
""")
def _cookie_secret_default(self):
if os.path.exists(self.cookie_secret_file):
with io.open(self.cookie_secret_file, 'rb') as f:
return f.read()
else:
secret = base64.encodestring(os.urandom(1024))
self._write_cookie_secret_file(secret)
return secret
def _write_cookie_secret_file(self, secret):
"""write my secret to my secret_file"""
self.log.info("Writing notebook server cookie secret to %s",
self.cookie_secret_file)
with io.open(self.cookie_secret_file, 'wb') as f:
f.write(secret)
try:
os.chmod(self.cookie_secret_file, 0o600)
except OSError:
self.log.warn("Could not set permissions on %s",
self.cookie_secret_file)
password = Unicode(u'',
config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
""")
open_browser = Bool(True,
config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'',
config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
webapp_settings = Dict(config=True,
help="DEPRECATED, use tornado_settings")
def _webapp_settings_changed(self, name, old, new):
self.log.warn(
"\n webapp_settings is deprecated, use tornado_settings.\n")
self.tornado_settings = new
tornado_settings = Dict(
config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
jinja_environment_options = Dict(
config=True,
help="Supply extra arguments that will be passed to Jinja environment."
)
enable_mathjax = Bool(
True,
config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
""")
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_url = Unicode('/',
config=True,
help='''The base URL for the notebook server.
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_url = '/' + new
elif not new.endswith('/'):
self.base_url = new + '/'
base_project_url = Unicode('/',
config=True,
help="""DEPRECATED use base_url""")
def _base_project_url_changed(self, name, old, new):
self.log.warn("base_project_url is deprecated, use base_url")
self.base_url = new
extra_static_paths = List(
Unicode,
config=True,
help="""Extra paths to search for serving static files.
This allows adding javascript/css to be available from the notebook server machine,
or overriding individual files in the IPython""")
def _extra_static_paths_default(self):
return [os.path.join(self.profile_dir.location, 'static')]
@property
def static_file_path(self):
"""return extra paths + the default location"""
return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
extra_template_paths = List(
Unicode,
config=True,
help="""Extra paths to search for serving jinja templates.
Can be used to override templates from IPython.html.templates.""")
def _extra_template_paths_default(self):
return []
@property
def template_file_path(self):
"""return extra paths + the default locations"""
return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST
extra_nbextensions_path = List(
Unicode,
config=True,
help="""extra paths to look for Javascript notebook extensions""")
@property
def nbextensions_path(self):
"""The path to look for Javascript notebook extensions"""
return self.extra_nbextensions_path + [
os.path.join(get_ipython_dir(), 'nbextensions')
] + SYSTEM_NBEXTENSIONS_DIRS
websocket_url = Unicode("",
config=True,
help="""The base URL for websockets,
if it differs from the HTTP server (hint: it almost certainly doesn't).
Should be in the form of an HTTP origin: ws[s]://hostname[:port]
""")
mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""")
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_url_prefix = self.tornado_settings.get(
"static_url_prefix", url_path_join(self.base_url, "static"))
# try local mathjax, either in nbextensions/mathjax or static/mathjax
for (url_prefix, search_path) in [
(url_path_join(self.base_url,
"nbextensions"), self.nbextensions_path),
(static_url_prefix, self.static_file_path),
]:
self.log.debug("searching for local mathjax in %s", search_path)
try:
mathjax = filefind(os.path.join('mathjax', 'MathJax.js'),
search_path)
except IOError:
continue
else:
url = url_path_join(url_prefix, u"mathjax/MathJax.js")
self.log.info("Serving local MathJax from %s at %s", mathjax,
url)
return url
# no local mathjax, serve from CDN
url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
contents_manager_class = Type(default_value=FileContentsManager,
klass=ContentsManager,
config=True,
help='The notebook manager class to use.')
kernel_manager_class = Type(default_value=MappingKernelManager,
config=True,
help='The kernel manager class to use.')
session_manager_class = Type(default_value=SessionManager,
config=True,
help='The session manager class to use.')
cluster_manager_class = Type(default_value=ClusterManager,
config=True,
help='The cluster manager class to use.')
config_manager_class = Type(default_value=ConfigManager,
config=True,
help='The config manager class to use')
kernel_spec_manager = Instance(KernelSpecManager)
def _kernel_spec_manager_default(self):
return KernelSpecManager(ipython_dir=self.ipython_dir)
kernel_spec_manager_class = Type(default_value=KernelSpecManager,
config=True,
help="""
The kernel spec manager class to use. Should be a subclass
of `IPython.kernel.kernelspec.KernelSpecManager`.
The Api of KernelSpecManager is provisional and might change
without warning between this version of IPython and the next stable one.
""")
login_handler_class = Type(
default_value=LoginHandler,
klass=web.RequestHandler,
config=True,
help='The login handler class to use.',
)
logout_handler_class = Type(
default_value=LogoutHandler,
klass=web.RequestHandler,
config=True,
help='The logout handler class to use.',
)
trust_xheaders = Bool(
False,
config=True,
help=
("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
"sent by the upstream reverse proxy. Necessary if the proxy handles SSL"
))
info_file = Unicode()
def _info_file_default(self):
info_file = "nbserver-%s.json" % os.getpid()
return os.path.join(self.profile_dir.security_dir, info_file)
pylab = Unicode('disabled',
config=True,
help="""
DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.
""")
def _pylab_changed(self, name, old, new):
"""when --pylab is specified, display a warning and exit"""
if new != 'warn':
backend = ' %s' % new
else:
backend = ''
self.log.error(
"Support for specifying --pylab on the command line has been removed."
)
self.log.error(
"Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself."
.format(backend))
self.exit(1)
notebook_dir = Unicode(
config=True, help="The directory to use for notebooks and kernels.")
def _notebook_dir_default(self):
if self.file_to_run:
return os.path.dirname(os.path.abspath(self.file_to_run))
else:
return py3compat.getcwd()
def _notebook_dir_changed(self, name, old, new):
"""Do a bit of validation of the notebook dir."""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
self.notebook_dir = os.path.abspath(new)
return
if not os.path.isdir(new):
raise TraitError("No such notebook dir: %r" % new)
# setting App.notebook_dir implies setting notebook and kernel dirs as well
self.config.FileContentsManager.root_dir = new
self.config.MappingKernelManager.root_dir = new
server_extensions = List(
Unicode(),
config=True,
help=(
"Python modules to load as notebook server extensions. "
"This is an experimental API, and may change in future releases."))
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if self.extra_args:
arg0 = self.extra_args[0]
f = os.path.abspath(arg0)
self.argv.remove(arg0)
if not os.path.exists(f):
self.log.critical("No such file or directory: %s", f)
self.exit(1)
# Use config here, to ensure that it takes higher priority than
# anything that comes from the profile.
c = Config()
if os.path.isdir(f):
c.NotebookApp.notebook_dir = f
elif os.path.isfile(f):
c.NotebookApp.file_to_run = f
self.update_config(c)
def init_kernel_argv(self):
"""add the profile-dir to arguments to be passed to IPython kernels"""
# FIXME: remove special treatment of IPython kernels
# Kernel should get *absolute* path to profile directory
self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location]
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_spec_manager = self.kernel_spec_manager_class(
ipython_dir=self.ipython_dir, )
self.kernel_manager = self.kernel_manager_class(
parent=self,
log=self.log,
ipython_kernel_argv=self.ipython_kernel_argv,
connection_dir=self.profile_dir.security_dir,
)
self.contents_manager = self.contents_manager_class(
parent=self,
log=self.log,
)
self.session_manager = self.session_manager_class(
parent=self,
log=self.log,
kernel_manager=self.kernel_manager,
contents_manager=self.contents_manager,
)
self.cluster_manager = self.cluster_manager_class(
parent=self,
log=self.log,
)
self.config_manager = self.config_manager_class(
parent=self,
log=self.log,
profile_dir=self.profile_dir.location,
)
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
for log in app_log, access_log, gen_log:
# consistent log output name (NotebookApp instead of tornado.access, etc.)
log.name = self.log.name
# hook up tornado 3's loggers to our app handlers
logger = logging.getLogger('tornado')
logger.propagate = True
logger.parent = self.log
logger.setLevel(self.log.level)
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.tornado_settings['allow_origin'] = self.allow_origin
if self.allow_origin_pat:
self.tornado_settings['allow_origin_pat'] = re.compile(
self.allow_origin_pat)
self.tornado_settings['allow_credentials'] = self.allow_credentials
self.web_app = NotebookWebApplication(
self, self.kernel_manager, self.contents_manager,
self.cluster_manager, self.session_manager,
self.kernel_spec_manager, self.config_manager, self.log,
self.base_url, self.default_url, self.tornado_settings,
self.jinja_environment_options)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.login_handler_class.validate_security(self,
ssl_options=ssl_options)
self.http_server = httpserver.HTTPServer(self.web_app,
ssl_options=ssl_options,
xheaders=self.trust_xheaders)
success = None
for port in random_ports(self.port, self.port_retries + 1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
if e.errno == errno.EADDRINUSE:
self.log.info(
'The port %i is already in use, trying another random port.'
% port)
continue
elif e.errno in (errno.EACCES,
getattr(errno, 'WSAEACCES', errno.EACCES)):
self.log.warn("Permission to listen on port %i denied" %
port)
continue
else:
raise
else:
self.port = port
success = True
break
if not success:
self.log.critical(
'ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
@property
def display_url(self):
ip = self.ip if self.ip else '[all ip addresses on your system]'
return self._url(ip)
@property
def connection_url(self):
ip = self.ip if self.ip else 'localhost'
return self._url(ip)
def _url(self, ip):
proto = 'https' if self.certfile else 'http'
return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url)
def init_terminals(self):
try:
from .terminal import initialize
initialize(self.web_app)
self.web_app.settings['terminals_available'] = True
except ImportError as e:
log = self.log.debug if sys.platform == 'win32' else self.log.warn
log("Terminals not available (error was %s)", e)
def init_signal(self):
if not sys.platform.startswith('win'):
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
if hasattr(signal, 'SIGUSR1'):
# Windows doesn't support SIGUSR1
signal.signal(signal.SIGUSR1, self._signal_info)
if hasattr(signal, 'SIGINFO'):
# only on BSD-based systems
signal.signal(signal.SIGINFO, self._signal_info)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
info = self.log.info
info('interrupted')
print(self.notebook_info())
sys.stdout.write("Shutdown this notebook server (y/[n])? ")
sys.stdout.flush()
r, w, x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y') and 'n' not in line.lower():
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.current().stop()
return
else:
print("No answer for 5s:", end=' ')
print("resuming operation...")
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.current().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.current().stop()
def _signal_info(self, sig, frame):
print(self.notebook_info())
def init_components(self):
"""Check the components submodule, and warn if it's unclean"""
status = submodule.check_submodule_status()
if status == 'missing':
self.log.warn(
"components submodule missing, running `git submodule update`")
submodule.update_submodules(submodule.ipython_parent())
elif status == 'unclean':
self.log.warn(
"components submodule unclean, you may see 404s on static/components"
)
self.log.warn(
"run `setup.py submodule` or `git submodule update` to update")
def init_server_extensions(self):
"""Load any extensions specified by config.
Import the module, then call the load_jupyter_server_extension function,
if one exists.
The extension API is experimental, and may change in future releases.
"""
for modulename in self.server_extensions:
try:
mod = importlib.import_module(modulename)
func = getattr(mod, 'load_jupyter_server_extension', None)
if func is not None:
func(self)
except Exception:
self.log.warn("Error loading server extension %s",
modulename,
exc_info=True)
@catch_config_error
def initialize(self, argv=None):
super(NotebookApp, self).initialize(argv)
self.init_logging()
self.init_kernel_argv()
self.init_configurables()
self.init_components()
self.init_webapp()
self.init_terminals()
self.init_signal()
self.init_server_extensions()
def cleanup_kernels(self):
"""Shutdown all kernels.
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
self.kernel_manager.shutdown_all()
def notebook_info(self):
"Return the current working directory and the server url information"
info = self.contents_manager.info_string() + "\n"
info += "%d active kernels \n" % len(self.kernel_manager._kernels)
return info + "The IPython Notebook is running at: %s" % self.display_url
def server_info(self):
"""Return a JSONable dict of information about this server."""
return {
'url': self.connection_url,
'hostname': self.ip if self.ip else 'localhost',
'port': self.port,
'secure': bool(self.certfile),
'base_url': self.base_url,
'notebook_dir': os.path.abspath(self.notebook_dir),
'pid': os.getpid()
}
def write_server_info_file(self):
"""Write the result of server_info() to the JSON file info_file."""
with open(self.info_file, 'w') as f:
json.dump(self.server_info(), f, indent=2)
def remove_server_info_file(self):
"""Remove the nbserver-<pid>.json file created for this server.
Ignores the error raised when the file has already been removed.
"""
try:
os.unlink(self.info_file)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def start(self):
""" Start the IPython Notebook server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
if self.subapp is not None:
return self.subapp.start()
info = self.log.info
for line in self.notebook_info().split("\n"):
info(line)
info(
"Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)."
)
self.write_server_info_file()
if self.open_browser or self.file_to_run:
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
if not os.path.exists(self.file_to_run):
self.log.critical("%s does not exist" % self.file_to_run)
self.exit(1)
relpath = os.path.relpath(self.file_to_run, self.notebook_dir)
uri = url_path_join('notebooks', *relpath.split(os.sep))
else:
uri = 'tree'
if browser:
b = lambda: browser.open(
url_path_join(self.connection_url, uri), new=2)
threading.Thread(target=b).start()
self.io_loop = ioloop.IOLoop.current()
try:
self.io_loop.start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
self.remove_server_info_file()
def stop(self):
def _stop():
self.http_server.stop()
self.io_loop.stop()
self.io_loop.add_callback(_stop)
class BaseDB(LoggingConfigurable):
"""Empty Parent class so traitlets work on DB."""
# base configurable traits:
session = Unicode("")
class InlineBackend(InlineBackendConfig):
"""An object to store configuration of the inline backend."""
def _config_changed(self, name, old, new):
# warn on change of renamed config section
if new.InlineBackendConfig != getattr(old, 'InlineBackendConfig',
Config()):
warn("InlineBackendConfig has been renamed to InlineBackend")
super(InlineBackend, self)._config_changed(name, old, new)
# The typical default figure size is too large for inline use,
# so we shrink the figure size to 6x4, and tweak fonts to
# make that fit.
rc = Dict(
{
'figure.figsize': (6.0, 4.0),
# play nicely with white background in the Qt and notebook frontend
'figure.facecolor': (1, 1, 1, 0),
'figure.edgecolor': (1, 1, 1, 0),
'axes.facecolor': (1, 1, 1, 0),
# 12pt labels get cutoff on 6x4 logplots, so use 10pt.
'font.size': 10,
# 72 dpi matches SVG/qtconsole
# this only affects PNG export, as SVG has no dpi setting
'savefig.dpi': 72,
# 10pt still needs a little more room on the xlabel:
'figure.subplot.bottom': .125
},
config=True,
help="""Subset of matplotlib rcParams that should be different for the
inline backend.""")
figure_formats = Set({'png'},
config=True,
help="""A set of figure formats to enable: 'png',
'retina', 'jpeg', 'svg', 'pdf'.""")
def _update_figure_formatters(self):
if self.shell is not None:
from IPython.core.pylabtools import select_figure_formats
select_figure_formats(self.shell, self.figure_formats,
**self.print_figure_kwargs)
def _figure_formats_changed(self, name, old, new):
if 'jpg' in new or 'jpeg' in new:
if not pil_available():
raise TraitError("Requires PIL/Pillow for JPG figures")
self._update_figure_formatters()
figure_format = Unicode(config=True,
help="""The figure format to enable (deprecated
use `figure_formats` instead)""")
def _figure_format_changed(self, name, old, new):
if new:
self.figure_formats = {new}
print_figure_kwargs = Dict(
{'bbox_inches': 'tight'},
config=True,
help="""Extra kwargs to be passed to fig.canvas.print_figure.
Logical examples include: bbox_inches, quality (for jpeg figures), etc.
""")
_print_figure_kwargs_changed = _update_figure_formatters
close_figures = Bool(True,
config=True,
help="""Close all figures at the end of each cell.
When True, ensures that each cell starts with no active figures, but it
also means that one must keep track of references in order to edit or
redraw figures in subsequent cells. This mode is ideal for the notebook,
where residual plots from other cells might be surprising.
When False, one must call figure() to create new figures. This means
that gcf() and getfigs() can reference figures created in other cells,
and the active figure can continue to be edited with pylab/pyplot
methods that reference the current active figure. This mode facilitates
iterative editing of figures, and behaves most consistently with
other matplotlib backends, but figure barriers between cells must
be explicit.
""")
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
class Kernel(SingletonConfigurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
def _eventloop_changed(self, name, old, new):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
session = Instance(Session, allow_none=True)
profile_dir = Instance('IPython.core.profiledir.ProfileDir',
allow_none=True)
shell_streams = List()
control_stream = Instance(ZMQStream, allow_none=True)
iopub_socket = Instance(zmq.Socket, allow_none=True)
stdin_socket = Instance(zmq.Socket, allow_none=True)
log = Instance(logging.Logger, allow_none=True)
# identities:
int_id = Integer(-1)
ident = Unicode()
def _ident_default(self):
return unicode_type(uuid.uuid4())
# This should be overridden by wrapper kernels that implement any real
# language.
language_info = {}
# any links that should go in the help menu
help_links = List()
# Private interface
_darwin_app_nap = Bool(
True,
config=True,
help="""Whether to use appnope for compatiblity with OS X App Nap.
Only affects OS X >= 10.9.
""")
# track associations with current request
_allow_stdin = Bool(False)
_parent_header = Dict()
_parent_ident = Any(b'')
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005, config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05, config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# set of aborted msg_ids
aborted = Set()
# Track execution count here. For IPython, we override this to use the
# execution count we store in the shell.
execution_count = 0
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Build dict of handlers for message types
msg_types = [
'execute_request',
'complete_request',
'inspect_request',
'history_request',
'kernel_info_request',
'connect_request',
'shutdown_request',
'apply_request',
'is_complete_request',
]
self.shell_handlers = {}
for msg_type in msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
control_msg_types = msg_types + ['clear_request', 'abort_request']
self.control_handlers = {}
for msg_type in control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if msg_id in self.aborted:
self.aborted.remove(msg_id)
# is it safe to assume a msg_id will not be resubmitted?
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
# ensure default_int_handler during handler call
sig = signal(SIGINT, default_int_handler)
self.log.debug("%s: %s", msg_type, msg)
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
signal(SIGINT, sig)
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop %s", self.eventloop)
for stream in self.shell_streams:
# flush any pending replies,
# which may be skipped by entering the eventloop
stream.flush(zmq.POLLOUT)
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
# publish idle status
self._publish_status('starting')
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _make_metadata(self, other=None):
"""init metadata dict, for execute/apply_reply"""
new_md = {
'dependencies_met': True,
'engine': self.ident,
'started': datetime.now(),
}
if other:
new_md.update(other)
return new_md
def _publish_execute_input(self, code, parent, execution_count):
"""Publish the code request on the iopub stream."""
self.session.send(self.iopub_socket,
u'execute_input', {
u'code': code,
u'execution_count': execution_count
},
parent=parent,
ident=self._topic('execute_input'))
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(
self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent or self._parent_header,
ident=self._topic('status'),
)
def set_parent(self, ident, parent):
"""Set the current parent_header
Side effects (IOPub messages) and replies are associated with
the request that caused them via the parent_header.
The parent identity is used to route input_request messages
on the stdin channel.
"""
self._parent_ident = ident
self._parent_header = parent
def send_response(self,
stream,
msg_or_type,
content=None,
ident=None,
buffers=None,
track=False,
header=None,
metadata=None):
"""Send a response to the message we're currently processing.
This accepts all the parameters of :meth:`ipython_kernel.session.Session.send`
except ``parent``.
This relies on :meth:`set_parent` having been called for the current
message.
"""
return self.session.send(stream, msg_or_type, content,
self._parent_header, ident, buffers, track,
header, metadata)
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
try:
content = parent[u'content']
code = py3compat.cast_unicode_py2(content[u'code'])
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
user_expressions = content.get('user_expressions', {})
allow_stdin = content.get('allow_stdin', False)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
stop_on_error = content.get('stop_on_error', True)
md = self._make_metadata(parent['metadata'])
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self.execution_count += 1
self._publish_execute_input(code, parent, self.execution_count)
reply_content = self.do_execute(code, silent, store_history,
user_expressions, allow_stdin)
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
md['status'] = reply_content['status']
if reply_content['status'] == 'error' and \
reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent,
metadata=md,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content'][
'status'] == u'error' and stop_on_error:
self._abort_queues()
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
"""Execute user code. Must be overridden by subclasses.
"""
raise NotImplementedError
def complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
cursor_pos = content['cursor_pos']
matches = self.do_complete(code, cursor_pos)
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply', matches,
parent, ident)
self.log.debug("%s", completion_msg)
def do_complete(self, code, cursor_pos):
"""Override in subclasses to find completions.
"""
return {
'matches': [],
'cursor_end': cursor_pos,
'cursor_start': cursor_pos,
'metadata': {},
'status': 'ok'
}
def inspect_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_inspect(content['code'], content['cursor_pos'],
content.get('detail_level', 0))
# Before we send this object over, we scrub it for JSON usage
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'inspect_reply', reply_content, parent,
ident)
self.log.debug("%s", msg)
def do_inspect(self, code, cursor_pos, detail_level=0):
"""Override in subclasses to allow introspection.
"""
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
def history_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_history(**content)
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'history_reply', reply_content, parent,
ident)
self.log.debug("%s", msg)
def do_history(self,
hist_access_type,
output,
raw,
session=None,
start=None,
stop=None,
n=None,
pattern=None,
unique=False):
"""Override in subclasses to access history.
"""
return {'history': []}
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
msg = self.session.send(stream, 'connect_reply', content, parent,
ident)
self.log.debug("%s", msg)
@property
def kernel_info(self):
return {
'protocol_version': release.kernel_protocol_version,
'implementation': self.implementation,
'implementation_version': self.implementation_version,
'language_info': self.language_info,
'banner': self.banner,
'help_links': self.help_links,
}
def kernel_info_request(self, stream, ident, parent):
msg = self.session.send(stream, 'kernel_info_reply', self.kernel_info,
parent, ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
content = self.do_shutdown(parent['content']['restart'])
self.session.send(stream,
u'shutdown_reply',
content,
parent,
ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply', content,
parent)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time() + 0.1, loop.stop)
def do_shutdown(self, restart):
"""Override in subclasses to do things when the frontend shuts down the
kernel.
"""
return {'status': 'ok', 'restart': restart}
def is_complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
reply_content = self.do_is_complete(code)
reply_content = json_clean(reply_content)
reply_msg = self.session.send(stream, 'is_complete_reply',
reply_content, parent, ident)
self.log.debug("%s", reply_msg)
def do_is_complete(self, code):
"""Override in subclasses to find completions.
"""
return {
'status': 'unknown',
}
#---------------------------------------------------------------------------
# Engine methods
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
md = self._make_metadata(parent['metadata'])
reply_content, result_buf = self.do_apply(content, bufs, msg_id, md)
# put 'ok'/'error' status in header, for scheduler introspection:
md['status'] = reply_content['status']
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
self.session.send(stream,
u'apply_reply',
reply_content,
parent=parent,
ident=ident,
buffers=result_buf,
metadata=md)
def do_apply(self, content, bufs, msg_id, reply_metadata):
"""Override in subclasses to support the IPython parallel framework.
"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Control messages
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specific msg by id"""
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, string_types):
msg_ids = [msg_ids]
if not msg_ids:
self._abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream,
'abort_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
content = self.do_clear()
self.session.send(stream,
'clear_reply',
ident=idents,
parent=parent,
content=content)
def do_clear(self):
"""Override in subclasses to clear the namespace
This is only required for IPython.parallel.
"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
if self.int_id >= 0:
base = "engine.%i" % self.int_id
else:
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def getpass(self, prompt=''):
"""Forward getpass to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"getpass was called, but this frontend does not support input requests."
)
return self._input_request(
prompt,
self._parent_ident,
self._parent_header,
password=True,
)
def raw_input(self, prompt=''):
"""Forward raw_input to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"raw_input was called, but this frontend does not support input requests."
)
return self._input_request(
prompt,
self._parent_ident,
self._parent_header,
password=False,
)
def _input_request(self, prompt, ident, parent, password=False):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# flush the stdin socket, to purge stale replies
while True:
try:
self.stdin_socket.recv_multipart(zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
break
else:
raise
# Send the input request.
content = json_clean(dict(prompt=prompt, password=password))
self.session.send(self.stdin_socket,
u'input_request',
content,
parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
except KeyboardInterrupt:
# re-raise KeyboardInterrupt, to truncate traceback
raise KeyboardInterrupt
else:
break
try:
value = py3compat.unicode_to_str(reply['content']['value'])
except:
self.log.error("Bad input_reply: %s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket,
self._shutdown_message,
ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[s.flush(zmq.POLLOUT) for s in self.shell_streams]
class FileContentsManager(ContentsManager):
root_dir = Unicode(config=True)
def _root_dir_default(self):
try:
return self.parent.notebook_dir
except AttributeError:
return getcwd()
@contextmanager
def perm_to_403(self, os_path=''):
"""context manager for turning permission errors into 403"""
try:
yield
except OSError as e:
if e.errno in {errno.EPERM, errno.EACCES}:
# make 403 error message without root prefix
# this may not work perfectly on unicode paths on Python 2,
# but nobody should be doing that anyway.
if not os_path:
os_path = str_to_unicode(e.filename or 'unknown file')
path = to_api_path(os_path, self.root_dir)
raise web.HTTPError(403, u'Permission denied: %s' % path)
else:
raise
@contextmanager
def open(self, os_path, *args, **kwargs):
"""wrapper around io.open that turns permission errors into 403"""
with self.perm_to_403(os_path):
with io.open(os_path, *args, **kwargs) as f:
yield f
@contextmanager
def atomic_writing(self, os_path, *args, **kwargs):
"""wrapper around atomic_writing that turns permission errors into 403"""
with self.perm_to_403(os_path):
with atomic_writing(os_path, *args, **kwargs) as f:
yield f
save_script = Bool(False, config=True, help='DEPRECATED, IGNORED')
def _save_script_changed(self):
self.log.warn("""
Automatically saving notebooks as scripts has been removed.
Use `ipython nbconvert --to python [notebook]` instead.
""")
def _root_dir_changed(self, name, old, new):
"""Do a bit of validation of the root_dir."""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
self.root_dir = os.path.abspath(new)
return
if not os.path.isdir(new):
raise TraitError("%r is not a directory" % new)
checkpoint_dir = Unicode(
'.ipynb_checkpoints',
config=True,
help="""The directory name in which to keep file checkpoints
This is a path relative to the file's own directory.
By default, it is .ipynb_checkpoints
""")
def _copy(self, src, dest):
"""copy src to dest
like shutil.copy2, but log errors in copystat
"""
shutil.copyfile(src, dest)
try:
shutil.copystat(src, dest)
except OSError as e:
self.log.debug("copystat on %s failed", dest, exc_info=True)
def _get_os_path(self, path):
"""Given an API path, return its file system path.
Parameters
----------
path : string
The relative API path to the named file.
Returns
-------
path : string
Native, absolute OS path to for a file.
"""
return to_os_path(path, self.root_dir)
def dir_exists(self, path):
"""Does the API-style path refer to an extant directory?
API-style wrapper for os.path.isdir
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
exists : bool
Whether the path is indeed a directory.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.isdir(os_path)
def is_hidden(self, path):
"""Does the API style path correspond to a hidden directory or file?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
hidden : bool
Whether the path exists and is hidden.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return is_hidden(os_path, self.root_dir)
def file_exists(self, path):
"""Returns True if the file exists, else returns False.
API-style wrapper for os.path.isfile
Parameters
----------
path : string
The relative path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the file exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path)
return os.path.isfile(os_path)
def exists(self, path):
"""Returns True if the path exists, else returns False.
API-style wrapper for os.path.exists
Parameters
----------
path : string
The API path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the target exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.exists(os_path)
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
os_path = self._get_os_path(path)
four_o_four = u'directory does not exist: %r' % path
if not os.path.isdir(os_path):
raise web.HTTPError(404, four_o_four)
elif is_hidden(os_path, self.root_dir):
self.log.info(
"Refusing to serve hidden directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
if content:
model['content'] = contents = []
os_dir = self._get_os_path(path)
for name in os.listdir(os_dir):
os_path = os.path.join(os_dir, name)
# skip over broken symlinks in listing
if not os.path.exists(os_path):
self.log.warn("%s doesn't exist", os_path)
continue
elif not os.path.isfile(os_path) and not os.path.isdir(
os_path):
self.log.debug("%s not a regular file", os_path)
continue
if self.should_list(name) and not is_hidden(
os_path, self.root_dir):
contents.append(
self.get(path='%s/%s' % (path, name), content=False))
model['format'] = 'json'
return model
def _file_model(self, path, content=True, format=None):
"""Build a model for a file
if content is requested, include the file contents.
format:
If 'text', the contents will be decoded as UTF-8.
If 'base64', the raw bytes contents will be encoded as base64.
If not specified, try to decode as UTF-8, and fall back to base64
"""
model = self._base_model(path)
model['type'] = 'file'
if content:
os_path = self._get_os_path(path)
if not os.path.isfile(os_path):
# could be FIFO
raise web.HTTPError(
400, "Cannot get content of non-file %s" % os_path)
with self.open(os_path, 'rb') as f:
bcontent = f.read()
if format != 'base64':
try:
model['content'] = bcontent.decode('utf8')
except UnicodeError as e:
if format == 'text':
raise web.HTTPError(400,
"%s is not UTF-8 encoded" % path)
else:
model['format'] = 'text'
if model['content'] is None:
model['content'] = base64.encodestring(bcontent).decode(
'ascii')
model['format'] = 'base64'
return model
def _notebook_model(self, path, content=True):
"""Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
model = self._base_model(path)
model['type'] = 'notebook'
if content:
os_path = self._get_os_path(path)
with self.open(os_path, 'r', encoding='utf-8') as f:
try:
nb = nbformat.read(f, as_version=4)
except Exception as e:
raise web.HTTPError(
400, u"Unreadable Notebook: %s %r" % (os_path, e))
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def get(self, path, content=True, type_=None, format=None):
""" Takes a path for an entity and returns its model
Parameters
----------
path : str
the API path that describes the relative path for the target
content : bool
Whether to include the contents in the reply
type_ : str, optional
The requested type - 'file', 'notebook', or 'directory'.
Will raise HTTPError 400 if the content doesn't match.
format : str, optional
The requested format for file contents. 'text' or 'base64'.
Ignored if this returns a notebook or directory model.
Returns
-------
model : dict
the contents model. If content=True, returns the contents
of the file or directory as well.
"""
path = path.strip('/')
if not self.exists(path):
raise web.HTTPError(404, u'No such file or directory: %s' % path)
os_path = self._get_os_path(path)
if os.path.isdir(os_path):
if type_ not in (None, 'directory'):
raise web.HTTPError(
400, u'%s is a directory, not a %s' % (path, type_))
model = self._dir_model(path, content=content)
elif type_ == 'notebook' or (type_ is None
and path.endswith('.ipynb')):
model = self._notebook_model(path, content=content)
else:
if type_ == 'directory':
raise web.HTTPError(400, u'%s is not a directory')
model = self._file_model(path, content=content, format=format)
return model
def _save_notebook(self, os_path, model, path=''):
"""save a notebook file"""
# Save the notebook file
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
with self.atomic_writing(os_path, encoding='utf-8') as f:
nbformat.write(nb, f, version=nbformat.NO_CONVERT)
def _save_file(self, os_path, model, path=''):
"""save a non-notebook file"""
fmt = model.get('format', None)
if fmt not in {'text', 'base64'}:
raise web.HTTPError(
400,
"Must specify format of file contents as 'text' or 'base64'")
try:
content = model['content']
if fmt == 'text':
bcontent = content.encode('utf8')
else:
b64_bytes = content.encode('ascii')
bcontent = base64.decodestring(b64_bytes)
except Exception as e:
raise web.HTTPError(400,
u'Encoding error saving %s: %s' % (os_path, e))
with self.atomic_writing(os_path, text=False) as f:
f.write(bcontent)
def _save_directory(self, os_path, model, path=''):
"""create a directory"""
if is_hidden(os_path, self.root_dir):
raise web.HTTPError(400,
u'Cannot create hidden directory %r' % os_path)
if not os.path.exists(os_path):
with self.perm_to_403():
os.mkdir(os_path)
elif not os.path.isdir(os_path):
raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
else:
self.log.debug("Directory %r already exists", os_path)
def save(self, model, path=''):
"""Save the file model and return the model with no content."""
path = path.strip('/')
if 'type' not in model:
raise web.HTTPError(400, u'No file type provided')
if 'content' not in model and model['type'] != 'directory':
raise web.HTTPError(400, u'No file content provided')
# One checkpoint should always exist
if self.file_exists(path) and not self.list_checkpoints(path):
self.create_checkpoint(path)
os_path = self._get_os_path(path)
self.log.debug("Saving %s", os_path)
try:
if model['type'] == 'notebook':
self._save_notebook(os_path, model, path)
elif model['type'] == 'file':
self._save_file(os_path, model, path)
elif model['type'] == 'directory':
self._save_directory(os_path, model, path)
else:
raise web.HTTPError(
400, "Unhandled contents type: %s" % model['type'])
except web.HTTPError:
raise
except Exception as e:
self.log.error(u'Error while saving file: %s %s',
path,
e,
exc_info=True)
raise web.HTTPError(
500, u'Unexpected error while saving file: %s %s' % (path, e))
validation_message = None
if model['type'] == 'notebook':
self.validate_notebook_model(model)
validation_message = model.get('message', None)
model = self.get(path, content=False)
if validation_message:
model['message'] = validation_message
return model
def update(self, model, path):
"""Update the file's path
For use in PATCH requests, to enable renaming a file without
re-uploading its contents. Only used for renaming at the moment.
"""
path = path.strip('/')
new_path = model.get('path', path).strip('/')
if path != new_path:
self.rename(path, new_path)
model = self.get(new_path, content=False)
return model
def delete(self, path):
"""Delete file at path."""
path = path.strip('/')
os_path = self._get_os_path(path)
rm = os.unlink
if os.path.isdir(os_path):
listing = os.listdir(os_path)
# don't delete non-empty directories (checkpoints dir doesn't count)
if listing and listing != [self.checkpoint_dir]:
raise web.HTTPError(400, u'Directory %s not empty' % os_path)
elif not os.path.isfile(os_path):
raise web.HTTPError(404, u'File does not exist: %s' % os_path)
# clear checkpoints
for checkpoint in self.list_checkpoints(path):
checkpoint_id = checkpoint['id']
cp_path = self.get_checkpoint_path(checkpoint_id, path)
if os.path.isfile(cp_path):
self.log.debug("Unlinking checkpoint %s", cp_path)
with self.perm_to_403():
rm(cp_path)
if os.path.isdir(os_path):
self.log.debug("Removing directory %s", os_path)
with self.perm_to_403():
shutil.rmtree(os_path)
else:
self.log.debug("Unlinking file %s", os_path)
with self.perm_to_403():
rm(os_path)
def rename(self, old_path, new_path):
"""Rename a file."""
old_path = old_path.strip('/')
new_path = new_path.strip('/')
if new_path == old_path:
return
new_os_path = self._get_os_path(new_path)
old_os_path = self._get_os_path(old_path)
# Should we proceed with the move?
if os.path.exists(new_os_path):
raise web.HTTPError(409, u'File already exists: %s' % new_path)
# Move the file
try:
with self.perm_to_403():
shutil.move(old_os_path, new_os_path)
except web.HTTPError:
raise
except Exception as e:
raise web.HTTPError(
500, u'Unknown error renaming file: %s %s' % (old_path, e))
# Move the checkpoints
old_checkpoints = self.list_checkpoints(old_path)
for cp in old_checkpoints:
checkpoint_id = cp['id']
old_cp_path = self.get_checkpoint_path(checkpoint_id, old_path)
new_cp_path = self.get_checkpoint_path(checkpoint_id, new_path)
if os.path.isfile(old_cp_path):
self.log.debug("Renaming checkpoint %s -> %s", old_cp_path,
new_cp_path)
with self.perm_to_403():
shutil.move(old_cp_path, new_cp_path)
# Checkpoint-related utilities
def get_checkpoint_path(self, checkpoint_id, path):
"""find the path to a checkpoint"""
path = path.strip('/')
parent, name = ('/' + path).rsplit('/', 1)
parent = parent.strip('/')
basename, ext = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=ext,
)
os_path = self._get_os_path(path=parent)
cp_dir = os.path.join(os_path, self.checkpoint_dir)
with self.perm_to_403():
ensure_dir_exists(cp_dir)
cp_path = os.path.join(cp_dir, filename)
return cp_path
def get_checkpoint_model(self, checkpoint_id, path):
"""construct the info dict for a given checkpoint"""
path = path.strip('/')
cp_path = self.get_checkpoint_path(checkpoint_id, path)
stats = os.stat(cp_path)
last_modified = tz.utcfromtimestamp(stats.st_mtime)
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
# public checkpoint API
def create_checkpoint(self, path):
"""Create a checkpoint from the current state of a file"""
path = path.strip('/')
if not self.file_exists(path):
raise web.HTTPError(404)
src_path = self._get_os_path(path)
# only the one checkpoint ID:
checkpoint_id = u"checkpoint"
cp_path = self.get_checkpoint_path(checkpoint_id, path)
self.log.debug("creating checkpoint for %s", path)
with self.perm_to_403():
self._copy(src_path, cp_path)
# return the checkpoint info
return self.get_checkpoint_model(checkpoint_id, path)
def list_checkpoints(self, path):
"""list the checkpoints for a given file
This contents manager currently only supports one checkpoint per file.
"""
path = path.strip('/')
checkpoint_id = "checkpoint"
os_path = self.get_checkpoint_path(checkpoint_id, path)
if not os.path.exists(os_path):
return []
else:
return [self.get_checkpoint_model(checkpoint_id, path)]
def restore_checkpoint(self, checkpoint_id, path):
"""restore a file to a checkpointed state"""
path = path.strip('/')
self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
nb_path = self._get_os_path(path)
cp_path = self.get_checkpoint_path(checkpoint_id, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(
404,
u'checkpoint does not exist: %s@%s' % (path, checkpoint_id))
# ensure notebook is readable (never restore from an unreadable notebook)
if cp_path.endswith('.ipynb'):
with self.open(cp_path, 'r', encoding='utf-8') as f:
nbformat.read(f, as_version=4)
self.log.debug("copying %s -> %s", cp_path, nb_path)
with self.perm_to_403():
self._copy(cp_path, nb_path)
def delete_checkpoint(self, checkpoint_id, path):
"""delete a file's checkpoint"""
path = path.strip('/')
cp_path = self.get_checkpoint_path(checkpoint_id, path)
if not os.path.isfile(cp_path):
raise web.HTTPError(
404,
u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id))
self.log.debug("unlinking %s", cp_path)
os.unlink(cp_path)
def info_string(self):
return "Serving notebooks from local directory: %s" % self.root_dir
def get_kernel_path(self, path, model=None):
"""Return the initial working dir a kernel associated with a given notebook"""
if '/' in path:
parent_dir = path.rsplit('/', 1)[0]
else:
parent_dir = ''
return self._get_os_path(parent_dir)
class Kernel(SessionFactory):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# kwargs:
exec_lines = List(Unicode, config=True, help="List of lines to execute")
# identities:
int_id = Integer(-1)
bident = CBytes()
ident = Unicode()
def _ident_changed(self, name, old, new):
self.bident = asbytes(new)
user_ns = Dict(config=True,
help="""Set the user's namespace of the Kernel""")
control_stream = Instance(zmqstream.ZMQStream)
task_stream = Instance(zmqstream.ZMQStream)
iopub_stream = Instance(zmqstream.ZMQStream)
client = Instance('IPython.parallel.Client')
# internals
shell_streams = List()
compiler = Instance(CommandCompiler, (), {})
completer = Instance(KernelCompleter)
aborted = Set()
shell_handlers = Dict()
control_handlers = Dict()
def _set_prefix(self):
self.prefix = "engine.%s" % self.int_id
def _connect_completer(self):
self.completer = KernelCompleter(self.user_ns)
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
self._set_prefix()
self._connect_completer()
self.on_trait_change(self._set_prefix, 'id')
self.on_trait_change(self._connect_completer, 'user_ns')
# Build dict of handlers for message types
for msg_type in [
'execute_request', 'complete_request', 'apply_request',
'clear_request'
]:
self.shell_handlers[msg_type] = getattr(self, msg_type)
for msg_type in ['shutdown_request', 'abort_request'
] + self.shell_handlers.keys():
self.control_handlers[msg_type] = getattr(self, msg_type)
self._initial_exec_lines()
def _wrap_exception(self, method=None):
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method=method)
content = wrap_exception(e_info)
return content
def _initial_exec_lines(self):
s = _Passer()
content = dict(silent=True, user_variable=[], user_expressions=[])
for line in self.exec_lines:
self.log.debug("executing initialization: %s" % line)
content.update({'code': line})
msg = self.session.msg('execute_request', content)
self.execute_request(s, [], msg)
#-------------------- control handlers -----------------------------
def abort_queues(self):
for stream in self.shell_streams:
if stream:
self.abort_queue(stream)
def abort_queue(self, stream):
while True:
idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info(str(msg))
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
# reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
# self.reply_socket.send(ident,zmq.SNDMORE)
# self.reply_socket.send_json(reply_msg)
reply_msg = self.session.send(stream,
reply_type,
content={'status': 'aborted'},
parent=msg,
ident=idents)
self.log.debug(str(reply_msg))
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
time.sleep(0.05)
def abort_request(self, stream, ident, parent):
"""abort a specifig msg by id"""
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, basestring):
msg_ids = [msg_ids]
if not msg_ids:
self.abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream,
'abort_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug(str(reply_msg))
def shutdown_request(self, stream, ident, parent):
"""kill ourself. This should really be handled in an external process"""
try:
self.abort_queues()
except:
content = self._wrap_exception('shutdown')
else:
content = dict(parent['content'])
content['status'] = 'ok'
msg = self.session.send(stream,
'shutdown_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug(str(msg))
dc = ioloop.DelayedCallback(lambda: sys.exit(0), 1000, self.loop)
dc.start()
def dispatch_control(self, msg):
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
else:
self.log.debug("Control received, %s", msg)
header = msg['header']
msg_id = header['msg_id']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r" % msg_type)
else:
handler(self.control_stream, idents, msg)
#-------------------- queue helpers ------------------------------
def check_dependencies(self, dependencies):
if not dependencies:
return True
if len(dependencies) == 2 and dependencies[0] in 'any all'.split():
anyorall = dependencies[0]
dependencies = dependencies[1]
else:
anyorall = 'all'
results = self.client.get_results(dependencies, status_only=True)
if results['status'] != 'ok':
return False
if anyorall == 'any':
if not results['completed']:
return False
else:
if results['pending']:
return False
return True
def check_aborted(self, msg_id):
return msg_id in self.aborted
#-------------------- queue handlers -----------------------------
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
self.user_ns = {}
msg = self.session.send(stream,
'clear_reply',
ident=idents,
parent=parent,
content=dict(status='ok'))
self._initial_exec_lines()
def execute_request(self, stream, ident, parent):
self.log.debug('execute request %s' % parent)
try:
code = parent[u'content'][u'code']
except:
self.log.error("Got bad msg: %s" % parent, exc_info=True)
return
self.session.send(self.iopub_stream,
u'pyin', {u'code': code},
parent=parent,
ident=asbytes('%s.pyin' % self.prefix))
started = datetime.now()
try:
comp_code = self.compiler(code, '<zmq-kernel>')
# allow for not overriding displayhook
if hasattr(sys.displayhook, 'set_parent'):
sys.displayhook.set_parent(parent)
sys.stdout.set_parent(parent)
sys.stderr.set_parent(parent)
exec comp_code in self.user_ns, self.user_ns
except:
exc_content = self._wrap_exception('execute')
# exc_msg = self.session.msg(u'pyerr', exc_content, parent)
self.session.send(self.iopub_stream,
u'pyerr',
exc_content,
parent=parent,
ident=asbytes('%s.pyerr' % self.prefix))
reply_content = exc_content
else:
reply_content = {'status': 'ok'}
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent=parent,
ident=ident,
subheader=dict(started=started))
self.log.debug(str(reply_msg))
if reply_msg['content']['status'] == u'error':
self.abort_queues()
def complete_request(self, stream, ident, parent):
matches = {'matches': self.complete(parent), 'status': 'ok'}
completion_msg = self.session.send(stream, 'complete_reply', matches,
parent, ident)
# print >> sys.__stdout__, completion_msg
def complete(self, msg):
return self.completer.complete(msg.content.line, msg.content.text)
def apply_request(self, stream, ident, parent):
# flush previous reply, so this request won't block it
stream.flush(zmq.POLLOUT)
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
# bound = parent['header'].get('bound', False)
except:
self.log.error("Got bad msg: %s" % parent, exc_info=True)
return
# pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
# self.iopub_stream.send(pyin_msg)
# self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent)
sub = {
'dependencies_met': True,
'engine': self.ident,
'started': datetime.now()
}
try:
# allow for not overriding displayhook
if hasattr(sys.displayhook, 'set_parent'):
sys.displayhook.set_parent(parent)
sys.stdout.set_parent(parent)
sys.stderr.set_parent(parent)
# exec "f(*args,**kwargs)" in self.user_ns, self.user_ns
working = self.user_ns
# suffix =
prefix = "_" + str(msg_id).replace("-", "") + "_"
f, args, kwargs = unpack_apply_message(bufs, working, copy=False)
# if bound:
# bound_ns = Namespace(working)
# args = [bound_ns]+list(args)
fname = getattr(f, '__name__', 'f')
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
ns = {fname: f, argname: args, kwargname: kwargs, resultname: None}
# print ns
working.update(ns)
code = "%s=%s(*%s,**%s)" % (resultname, fname, argname, kwargname)
try:
exec code in working, working
result = working.get(resultname)
finally:
for key in ns.iterkeys():
working.pop(key)
# if bound:
# working.update(bound_ns)
packed_result, buf = serialize_object(result)
result_buf = [packed_result] + buf
except:
exc_content = self._wrap_exception('apply')
# exc_msg = self.session.msg(u'pyerr', exc_content, parent)
self.session.send(self.iopub_stream,
u'pyerr',
exc_content,
parent=parent,
ident=asbytes('%s.pyerr' % self.prefix))
reply_content = exc_content
result_buf = []
if exc_content['ename'] == 'UnmetDependency':
sub['dependencies_met'] = False
else:
reply_content = {'status': 'ok'}
# put 'ok'/'error' status in header, for scheduler introspection:
sub['status'] = reply_content['status']
reply_msg = self.session.send(stream,
u'apply_reply',
reply_content,
parent=parent,
ident=ident,
buffers=result_buf,
subheader=sub)
# flush i/o
# should this be before reply_msg is sent, like in the single-kernel code,
# or should nothing get in the way of real results?
sys.stdout.flush()
sys.stderr.flush()
def dispatch_queue(self, stream, msg):
self.control_stream.flush()
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
else:
self.log.debug("Message received, %s", msg)
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
if self.check_aborted(msg_id):
self.aborted.remove(msg_id)
# is it safe to assume a msg_id will not be resubmitted?
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
reply_msg = self.session.send(stream,
reply_type,
subheader=status,
content=status,
parent=msg,
ident=idents)
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r" % msg_type)
else:
handler(stream, idents, msg)
def start(self):
#### stream mode:
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
self.control_stream.on_err(printer)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_queue(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
s.on_err(printer)
if self.iopub_stream:
self.iopub_stream.on_err(printer)
class HistoryAccessor(HistoryAccessorBase):
"""Access the history database without adding to it.
This is intended for use by standalone history tools. IPython shells use
HistoryManager, below, which is a subclass of this."""
# String holding the path to the history file
hist_file = Unicode(
config=True,
help="""Path to file to use for SQLite history database.
By default, IPython will put the history database in the IPython
profile directory. If you would rather share one history among
profiles, you can set this value in each, so that they are consistent.
Due to an issue with fcntl, SQLite is known to misbehave on some NFS
mounts. If you see IPython hanging, try setting this to something on a
local disk, e.g::
ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite
""")
enabled = Bool(True,
config=True,
help="""enable the SQLite history
set enabled=False to disable the SQLite history,
in which case there will be no stored history, no SQLite connection,
and no background saving thread. This may be necessary in some
threaded environments where IPython is embedded.
""")
connection_options = Dict(
config=True,
help="""Options for configuring the SQLite connection
These options are passed as keyword args to sqlite3.connect
when establishing database conenctions.
""")
# The SQLite database
db = Any()
def _db_changed(self, name, old, new):
"""validate the db, since it can be an Instance of two different types"""
connection_types = (DummyDB, )
if sqlite3 is not None:
connection_types = (DummyDB, sqlite3.Connection)
if not isinstance(new, connection_types):
msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \
(self.__class__.__name__, new)
raise TraitError(msg)
def __init__(self, profile='default', hist_file=u'', **traits):
"""Create a new history accessor.
Parameters
----------
profile : str
The name of the profile from which to open history.
hist_file : str
Path to an SQLite history database stored by IPython. If specified,
hist_file overrides profile.
config : :class:`~IPython.config.loader.Config`
Config object. hist_file can also be set through this.
"""
# We need a pointer back to the shell for various tasks.
super(HistoryAccessor, self).__init__(**traits)
# defer setting hist_file from kwarg until after init,
# otherwise the default kwarg value would clobber any value
# set by config
if hist_file:
self.hist_file = hist_file
if self.hist_file == u'':
# No one has set the hist_file, yet.
self.hist_file = self._get_hist_file_name(profile)
if sqlite3 is None and self.enabled:
warn(
"IPython History requires SQLite, your history will not be saved"
)
self.enabled = False
self.init_db()
def _get_hist_file_name(self, profile='default'):
"""Find the history file for the given profile name.
This is overridden by the HistoryManager subclass, to use the shell's
active profile.
Parameters
----------
profile : str
The name of a profile which has a history file.
"""
return os.path.join(locate_profile(profile), 'history.sqlite')
@catch_corrupt_db
def init_db(self):
"""Connect to the database, and create tables if necessary."""
if not self.enabled:
self.db = DummyDB()
return
# use detect_types so that timestamps return datetime objects
kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES
| sqlite3.PARSE_COLNAMES)
kwargs.update(self.connection_options)
self.db = sqlite3.connect(self.hist_file, **kwargs)
self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
primary key autoincrement, start timestamp,
end timestamp, num_cmds integer, remark text)""")
self.db.execute("""CREATE TABLE IF NOT EXISTS history
(session integer, line integer, source text, source_raw text,
PRIMARY KEY (session, line))""")
# Output history is optional, but ensure the table's there so it can be
# enabled later.
self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
(session integer, line integer, output text,
PRIMARY KEY (session, line))""")
self.db.commit()
def writeout_cache(self):
"""Overridden by HistoryManager to dump the cache before certain
database lookups."""
pass
## -------------------------------
## Methods for retrieving history:
## -------------------------------
def _run_sql(self, sql, params, raw=True, output=False):
"""Prepares and runs an SQL query for the history database.
Parameters
----------
sql : str
Any filtering expressions to go after SELECT ... FROM ...
params : tuple
Parameters passed to the SQL query (to replace "?")
raw, output : bool
See :meth:`get_range`
Returns
-------
Tuples as :meth:`get_range`
"""
toget = 'source_raw' if raw else 'source'
sqlfrom = "history"
if output:
sqlfrom = "history LEFT JOIN output_history USING (session, line)"
toget = "history.%s, output_history.output" % toget
cur = self.db.execute("SELECT session, line, %s FROM %s " %\
(toget, sqlfrom) + sql, params)
if output: # Regroup into 3-tuples, and parse JSON
return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
return cur
@needs_sqlite
@catch_corrupt_db
def get_session_info(self, session):
"""Get info about a session.
Parameters
----------
session : int
Session number to retrieve.
Returns
-------
session_id : int
Session ID number
start : datetime
Timestamp for the start of the session.
end : datetime
Timestamp for the end of the session, or None if IPython crashed.
num_cmds : int
Number of commands run, or None if IPython crashed.
remark : unicode
A manually set description.
"""
query = "SELECT * from sessions where session == ?"
return self.db.execute(query, (session, )).fetchone()
@catch_corrupt_db
def get_last_session_id(self):
"""Get the last session ID currently in the database.
Within IPython, this should be the same as the value stored in
:attr:`HistoryManager.session_number`.
"""
for record in self.get_tail(n=1, include_latest=True):
return record[0]
@catch_corrupt_db
def get_tail(self, n=10, raw=True, output=False, include_latest=False):
"""Get the last n lines from the history database.
Parameters
----------
n : int
The number of lines to get
raw, output : bool
See :meth:`get_range`
include_latest : bool
If False (default), n+1 lines are fetched, and the latest one
is discarded. This is intended to be used where the function
is called by a user command, which it should not return.
Returns
-------
Tuples as :meth:`get_range`
"""
self.writeout_cache()
if not include_latest:
n += 1
cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?", (n, ),
raw=raw,
output=output)
if not include_latest:
return reversed(list(cur)[1:])
return reversed(list(cur))
@catch_corrupt_db
def search(self,
pattern="*",
raw=True,
search_raw=True,
output=False,
n=None,
unique=False):
"""Search the database using unix glob-style matching (wildcards
* and ?).
Parameters
----------
pattern : str
The wildcarded pattern to match when searching
search_raw : bool
If True, search the raw input, otherwise, the parsed input
raw, output : bool
See :meth:`get_range`
n : None or int
If an integer is given, it defines the limit of
returned entries.
unique : bool
When it is true, return only unique entries.
Returns
-------
Tuples as :meth:`get_range`
"""
tosearch = "source_raw" if search_raw else "source"
if output:
tosearch = "history." + tosearch
self.writeout_cache()
sqlform = "WHERE %s GLOB ?" % tosearch
params = (pattern, )
if unique:
sqlform += ' GROUP BY {0}'.format(tosearch)
if n is not None:
sqlform += " ORDER BY session DESC, line DESC LIMIT ?"
params += (n, )
elif unique:
sqlform += " ORDER BY session, line"
cur = self._run_sql(sqlform, params, raw=raw, output=output)
if n is not None:
return reversed(list(cur))
return cur
@catch_corrupt_db
def get_range(self, session, start=1, stop=None, raw=True, output=False):
"""Retrieve input by session.
Parameters
----------
session : int
Session number to retrieve.
start : int
First line to retrieve.
stop : int
End of line range (excluded from output itself). If None, retrieve
to the end of the session.
raw : bool
If True, return untranslated input
output : bool
If True, attempt to include output. This will be 'real' Python
objects for the current session, or text reprs from previous
sessions if db_log_output was enabled at the time. Where no output
is found, None is used.
Returns
-------
entries
An iterator over the desired lines. Each line is a 3-tuple, either
(session, line, input) if output is False, or
(session, line, (input, output)) if output is True.
"""
if stop:
lineclause = "line >= ? AND line < ?"
params = (session, start, stop)
else:
lineclause = "line>=?"
params = (session, start)
return self._run_sql("WHERE session==? AND %s" % lineclause,
params,
raw=raw,
output=output)
def get_range_by_str(self, rangestr, raw=True, output=False):
"""Get lines of history from a string of ranges, as used by magic
commands %hist, %save, %macro, etc.
Parameters
----------
rangestr : str
A string specifying ranges, e.g. "5 ~2/1-4". See
:func:`magic_history` for full details.
raw, output : bool
As :meth:`get_range`
Returns
-------
Tuples as :meth:`get_range`
"""
for sess, s, e in extract_hist_ranges(rangestr):
for line in self.get_range(sess, s, e, raw=raw, output=output):
yield line
class WinHPCJob(Configurable):
job_id = Unicode('')
job_name = Unicode('MyJob', config=True)
min_cores = Int(1, config=True)
max_cores = Int(1, config=True)
min_sockets = Int(1, config=True)
max_sockets = Int(1, config=True)
min_nodes = Int(1, config=True)
max_nodes = Int(1, config=True)
unit_type = Unicode("Core", config=True)
auto_calculate_min = Bool(True, config=True)
auto_calculate_max = Bool(True, config=True)
run_until_canceled = Bool(False, config=True)
is_exclusive = Bool(False, config=True)
username = Unicode(find_username(), config=True)
job_type = Unicode('Batch', config=True)
priority = Enum(
('Lowest', 'BelowNormal', 'Normal', 'AboveNormal', 'Highest'),
default_value='Highest',
config=True)
requested_nodes = Unicode('', config=True)
project = Unicode('IPython', config=True)
xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/')
version = Unicode("2.000")
tasks = List([])
@property
def owner(self):
return self.username
def _write_attr(self, root, attr, key):
s = as_str(getattr(self, attr, ''))
if s:
root.set(key, s)
def as_element(self):
# We have to add _A_ type things to get the right order than
# the MSFT XML parser expects.
root = ET.Element('Job')
self._write_attr(root, 'version', '_A_Version')
self._write_attr(root, 'job_name', '_B_Name')
self._write_attr(root, 'unit_type', '_C_UnitType')
self._write_attr(root, 'min_cores', '_D_MinCores')
self._write_attr(root, 'max_cores', '_E_MaxCores')
self._write_attr(root, 'min_sockets', '_F_MinSockets')
self._write_attr(root, 'max_sockets', '_G_MaxSockets')
self._write_attr(root, 'min_nodes', '_H_MinNodes')
self._write_attr(root, 'max_nodes', '_I_MaxNodes')
self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
self._write_attr(root, 'username', '_L_UserName')
self._write_attr(root, 'job_type', '_M_JobType')
self._write_attr(root, 'priority', '_N_Priority')
self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
self._write_attr(root, 'project', '_R_Project')
self._write_attr(root, 'owner', '_S_Owner')
self._write_attr(root, 'xmlns', '_T_xmlns')
dependencies = ET.SubElement(root, "Dependencies")
etasks = ET.SubElement(root, "Tasks")
for t in self.tasks:
etasks.append(t.as_element())
return root
def tostring(self):
"""Return the string representation of the job description XML."""
root = self.as_element()
indent(root)
txt = ET.tostring(root, encoding="utf-8")
# Now remove the tokens used to order the attributes.
txt = re.sub(r'_[A-Z]_', '', txt)
txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
return txt
def write(self, filename):
"""Write the XML job description to a file."""
txt = self.tostring()
with open(filename, 'w') as f:
f.write(txt)
def add_task(self, task):
"""Add a task to the job.
Parameters
----------
task : :class:`WinHPCTask`
The task object to add.
"""
self.tasks.append(task)
class TerminalInteractiveShell(InteractiveShell):
autoedit_syntax = CBool(False,
config=True,
help="auto editing of files with syntax errors.")
banner = Unicode('')
banner1 = Unicode(
default_banner,
config=True,
help="""The part of the banner to be printed before the profile""")
banner2 = Unicode(
'',
config=True,
help="""The part of the banner to be printed after the profile""")
confirm_exit = CBool(
True,
config=True,
help="""
Set to confirm when you try to exit IPython with an EOF (Control-D
in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
you can force a direct exit without any confirmation.""",
)
# This display_banner only controls whether or not self.show_banner()
# is called when mainloop/interact are called. The default is False
# because for the terminal based application, the banner behavior
# is controlled by Global.display_banner, which IPythonApp looks at
# to determine if *it* should call show_banner() by hand or not.
display_banner = CBool(False) # This isn't configurable!
embedded = CBool(False)
embedded_active = CBool(False)
editor = Unicode(
get_default_editor(),
config=True,
help="Set the editor used by IPython (default to $EDITOR/vi/notepad).")
pager = Unicode('less',
config=True,
help="The shell program to be used for paging.")
screen_length = Integer(
0,
config=True,
help="""Number of lines of your screen, used to control printing of very
long strings. Strings longer than this number of lines will be sent
through a pager instead of directly printed. The default value for
this is 0, which means IPython will auto-detect your screen size every
time it needs to print certain potentially long strings (this doesn't
change the behavior of the 'print' keyword, it's only triggered
internally). If for some reason this isn't working well (it needs
curses support), specify it yourself. Otherwise don't change the
default.""",
)
term_title = CBool(False,
config=True,
help="Enable auto setting the terminal title.")
# This `using_paste_magics` is used to detect whether the code is being
# executed via paste magics functions
using_paste_magics = CBool(False)
# In the terminal, GUI control is done via PyOS_InputHook
@staticmethod
def enable_gui(gui=None, app=None):
"""Switch amongst GUI input hooks by name.
"""
# Deferred import
from IPython.lib.inputhook import enable_gui as real_enable_gui
return real_enable_gui(gui, app)
def __init__(self,
config=None,
ipython_dir=None,
profile_dir=None,
user_ns=None,
user_module=None,
custom_exceptions=((), None),
usage=None,
banner1=None,
banner2=None,
display_banner=None,
**kwargs):
super(TerminalInteractiveShell,
self).__init__(config=config,
ipython_dir=ipython_dir,
profile_dir=profile_dir,
user_ns=user_ns,
user_module=user_module,
custom_exceptions=custom_exceptions,
**kwargs)
# use os.system instead of utils.process.system by default,
# because piped system doesn't make sense in the Terminal:
self.system = self.system_raw
self.init_term_title()
self.init_usage(usage)
self.init_banner(banner1, banner2, display_banner)
#-------------------------------------------------------------------------
# Overrides of init stages
#-------------------------------------------------------------------------
def init_display_formatter(self):
super(TerminalInteractiveShell, self).init_display_formatter()
# terminal only supports plaintext
self.display_formatter.active_types = ['text/plain']
#-------------------------------------------------------------------------
# Things related to the terminal
#-------------------------------------------------------------------------
@property
def usable_screen_length(self):
if self.screen_length == 0:
return 0
else:
num_lines_bot = self.separate_in.count('\n') + 1
return self.screen_length - num_lines_bot
def init_term_title(self):
# Enable or disable the terminal title.
if self.term_title:
toggle_set_term_title(True)
set_term_title('IPython: ' + abbrev_cwd())
else:
toggle_set_term_title(False)
#-------------------------------------------------------------------------
# Things related to aliases
#-------------------------------------------------------------------------
def init_alias(self):
# The parent class defines aliases that can be safely used with any
# frontend.
super(TerminalInteractiveShell, self).init_alias()
# Now define aliases that only make sense on the terminal, because they
# need direct access to the console in a way that we can't emulate in
# GUI or web frontend
if os.name == 'posix':
aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'),
('man', 'man')]
elif os.name == 'nt':
aliases = [('cls', 'cls')]
for name, cmd in aliases:
self.alias_manager.define_alias(name, cmd)
#-------------------------------------------------------------------------
# Things related to the banner and usage
#-------------------------------------------------------------------------
def _banner1_changed(self):
self.compute_banner()
def _banner2_changed(self):
self.compute_banner()
def _term_title_changed(self, name, new_value):
self.init_term_title()
def init_banner(self, banner1, banner2, display_banner):
if banner1 is not None:
self.banner1 = banner1
if banner2 is not None:
self.banner2 = banner2
if display_banner is not None:
self.display_banner = display_banner
self.compute_banner()
def show_banner(self, banner=None):
if banner is None:
banner = self.banner
self.write(banner)
def compute_banner(self):
self.banner = self.banner1
if self.profile and self.profile != 'default':
self.banner += '\nIPython profile: %s\n' % self.profile
if self.banner2:
self.banner += '\n' + self.banner2
def init_usage(self, usage=None):
if usage is None:
self.usage = interactive_usage
else:
self.usage = usage
#-------------------------------------------------------------------------
# Mainloop and code execution logic
#-------------------------------------------------------------------------
def mainloop(self, display_banner=None):
"""Start the mainloop.
If an optional banner argument is given, it will override the
internally created default banner.
"""
with nested(self.builtin_trap, self.display_trap):
while 1:
try:
self.interact(display_banner=display_banner)
#self.interact_with_readline()
# XXX for testing of a readline-decoupled repl loop, call
# interact_with_readline above
break
except KeyboardInterrupt:
# this should not be necessary, but KeyboardInterrupt
# handling seems rather unpredictable...
self.write("\nKeyboardInterrupt in interact()\n")
def _replace_rlhist_multiline(self, source_raw, hlen_before_cell):
"""Store multiple lines as a single entry in history"""
# do nothing without readline or disabled multiline
if not self.has_readline or not self.multiline_history:
return hlen_before_cell
# windows rl has no remove_history_item
if not hasattr(self.readline, "remove_history_item"):
return hlen_before_cell
# skip empty cells
if not source_raw.rstrip():
return hlen_before_cell
# nothing changed do nothing, e.g. when rl removes consecutive dups
hlen = self.readline.get_current_history_length()
if hlen == hlen_before_cell:
return hlen_before_cell
for i in range(hlen - hlen_before_cell):
self.readline.remove_history_item(hlen - i - 1)
stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')
self.readline.add_history(
py3compat.unicode_to_str(source_raw.rstrip(), stdin_encoding))
return self.readline.get_current_history_length()
def interact(self, display_banner=None):
"""Closely emulate the interactive Python console."""
# batch run -> do not interact
if self.exit_now:
return
if display_banner is None:
display_banner = self.display_banner
if isinstance(display_banner, basestring):
self.show_banner(display_banner)
elif display_banner:
self.show_banner()
more = False
if self.has_readline:
self.readline_startup_hook(self.pre_readline)
hlen_b4_cell = self.readline.get_current_history_length()
else:
hlen_b4_cell = 0
# exit_now is set by a call to %Exit or %Quit, through the
# ask_exit callback.
while not self.exit_now:
self.hooks.pre_prompt_hook()
if more:
try:
prompt = self.prompt_manager.render('in2')
except:
self.showtraceback()
if self.autoindent:
self.rl_do_indent = True
else:
try:
prompt = self.separate_in + self.prompt_manager.render(
'in')
except:
self.showtraceback()
try:
line = self.raw_input(prompt)
if self.exit_now:
# quick exit on sys.std[in|out] close
break
if self.autoindent:
self.rl_do_indent = False
except KeyboardInterrupt:
#double-guard against keyboardinterrupts during kbdint handling
try:
self.write('\nKeyboardInterrupt\n')
source_raw = self.input_splitter.source_raw_reset()[1]
hlen_b4_cell = \
self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
more = False
except KeyboardInterrupt:
pass
except EOFError:
if self.autoindent:
self.rl_do_indent = False
if self.has_readline:
self.readline_startup_hook(None)
self.write('\n')
self.exit()
except bdb.BdbQuit:
warn(
'The Python debugger has exited with a BdbQuit exception.\n'
'Because of how pdb handles the stack, it is impossible\n'
'for IPython to properly format this particular exception.\n'
'IPython will resume normal operation.')
except:
# exceptions here are VERY RARE, but they can be triggered
# asynchronously by signal handlers, for example.
self.showtraceback()
else:
self.input_splitter.push(line)
more = self.input_splitter.push_accepts_more()
if (self.SyntaxTB.last_syntax_error and self.autoedit_syntax):
self.edit_syntax_error()
if not more:
source_raw = self.input_splitter.source_raw_reset()[1]
self.run_cell(source_raw, store_history=True)
hlen_b4_cell = \
self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
# Turn off the exit flag, so the mainloop can be restarted if desired
self.exit_now = False
def raw_input(self, prompt=''):
"""Write a prompt and read a line.
The returned line does not include the trailing newline.
When the user enters the EOF key sequence, EOFError is raised.
Optional inputs:
- prompt(''): a string to be printed to prompt the user.
- continue_prompt(False): whether this line is the first one or a
continuation in a sequence of inputs.
"""
# Code run by the user may have modified the readline completer state.
# We must ensure that our completer is back in place.
if self.has_readline:
self.set_readline_completer()
# raw_input expects str, but we pass it unicode sometimes
prompt = py3compat.cast_bytes_py2(prompt)
try:
line = py3compat.str_to_unicode(self.raw_input_original(prompt))
except ValueError:
warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
" or sys.stdout.close()!\nExiting IPython!\n")
self.ask_exit()
return ""
# Try to be reasonably smart about not re-indenting pasted input more
# than necessary. We do this by trimming out the auto-indent initial
# spaces, if the user's actual input started itself with whitespace.
if self.autoindent:
if num_ini_spaces(line) > self.indent_current_nsp:
line = line[self.indent_current_nsp:]
self.indent_current_nsp = 0
return line
#-------------------------------------------------------------------------
# Methods to support auto-editing of SyntaxErrors.
#-------------------------------------------------------------------------
def edit_syntax_error(self):
"""The bottom half of the syntax error handler called in the main loop.
Loop until syntax error is fixed or user cancels.
"""
while self.SyntaxTB.last_syntax_error:
# copy and clear last_syntax_error
err = self.SyntaxTB.clear_err_state()
if not self._should_recompile(err):
return
try:
# may set last_syntax_error again if a SyntaxError is raised
self.safe_execfile(err.filename, self.user_ns)
except:
self.showtraceback()
else:
try:
f = open(err.filename)
try:
# This should be inside a display_trap block and I
# think it is.
sys.displayhook(f.read())
finally:
f.close()
except:
self.showtraceback()
def _should_recompile(self, e):
"""Utility routine for edit_syntax_error"""
if e.filename in ('<ipython console>', '<input>', '<string>',
'<console>', '<BackgroundJob compilation>', None):
return False
try:
if (self.autoedit_syntax and not self.ask_yes_no(
'Return to editor to correct syntax error? '
'[Y/n] ', 'y')):
return False
except EOFError:
return False
def int0(x):
try:
return int(x)
except TypeError:
return 0
# always pass integer line and offset values to editor hook
try:
self.hooks.fix_error_editor(e.filename, int0(e.lineno),
int0(e.offset), e.msg)
except TryNext:
warn('Could not open editor')
return False
return True
#-------------------------------------------------------------------------
# Things related to exiting
#-------------------------------------------------------------------------
def ask_exit(self):
""" Ask the shell to exit. Can be overiden and used as a callback. """
self.exit_now = True
def exit(self):
"""Handle interactive exit.
This method calls the ask_exit callback."""
if self.confirm_exit:
if self.ask_yes_no('Do you really want to exit ([y]/n)?', 'y'):
self.ask_exit()
else:
self.ask_exit()
#-------------------------------------------------------------------------
# Things related to magics
#-------------------------------------------------------------------------
def init_magics(self):
super(TerminalInteractiveShell, self).init_magics()
self.register_magics(TerminalMagics)
def showindentationerror(self):
super(TerminalInteractiveShell, self).showindentationerror()
if not self.using_paste_magics:
print("If you want to paste code into IPython, try the "
"%paste and %cpaste magic functions.")
class MongoDB(BaseDB):
"""MongoDB TaskRecord backend."""
connection_args = List(
config=True,
help="""Positional arguments to be passed to pymongo.Connection. Only
necessary if the default mongodb configuration does not point to your
mongod instance.""")
connection_kwargs = Dict(
config=True,
help="""Keyword arguments to be passed to pymongo.Connection. Only
necessary if the default mongodb configuration does not point to your
mongod instance.""")
database = Unicode(
"ipython-tasks",
config=True,
help=
"""The MongoDB database name to use for storing tasks for this session. If unspecified,
a new database will be created with the Hub's IDENT. Specifying the database will result
in tasks from previous sessions being available via Clients' db_query and
get_result methods.""")
_connection = Instance(Connection) # pymongo connection
def __init__(self, **kwargs):
super(MongoDB, self).__init__(**kwargs)
if self._connection is None:
self._connection = Connection(*self.connection_args,
**self.connection_kwargs)
if not self.database:
self.database = self.session
self._db = self._connection[self.database]
self._records = self._db['task_records']
self._records.ensure_index('msg_id', unique=True)
self._records.ensure_index('submitted') # for sorting history
# for rec in self._records.find
def _binary_buffers(self, rec):
for key in ('buffers', 'result_buffers'):
if rec.get(key, None):
rec[key] = map(Binary, rec[key])
return rec
def add_record(self, msg_id, rec):
"""Add a new Task Record, by msg_id."""
# print rec
rec = self._binary_buffers(rec)
self._records.insert(rec)
def get_record(self, msg_id):
"""Get a specific Task Record, by msg_id."""
r = self._records.find_one({'msg_id': msg_id})
if not r:
# r will be '' if nothing is found
raise KeyError(msg_id)
return r
def update_record(self, msg_id, rec):
"""Update the data in an existing record."""
rec = self._binary_buffers(rec)
self._records.update({'msg_id': msg_id}, {'$set': rec})
def drop_matching_records(self, check):
"""Remove a record from the DB."""
self._records.remove(check)
def drop_record(self, msg_id):
"""Remove a record from the DB."""
self._records.remove({'msg_id': msg_id})
def find_records(self, check, keys=None):
"""Find records matching a query dict, optionally extracting subset of keys.
Returns list of matching records.
Parameters
----------
check: dict
mongodb-style query argument
keys: list of strs [optional]
if specified, the subset of keys to extract. msg_id will *always* be
included.
"""
if keys and 'msg_id' not in keys:
keys.append('msg_id')
matches = list(self._records.find(check, keys))
for rec in matches:
rec.pop('_id')
return matches
def get_history(self):
"""get all msg_ids, ordered by time submitted."""
cursor = self._records.find({}, {'msg_id': 1}).sort('submitted')
return [rec['msg_id'] for rec in cursor]
