class SanitizeHTML(Preprocessor):
# Bleach config.
attributes = Any(
config=True,
default_value=ALLOWED_ATTRIBUTES,
help="Allowed HTML tag attributes",
)
tags = List(
Unicode(),
config=True,
default_value=ALLOWED_TAGS,
help="List of HTML tags to allow",
)
styles = List(
Unicode(),
config=True,
default_value=ALLOWED_STYLES,
help="Allowed CSS styles if <style> tag is whitelisted"
)
strip = Bool(
config=True,
default_value=False,
help="If True, remove unsafe markup entirely instead of escaping"
)
strip_comments = Bool(
config=True,
default_value=True,
help="If True, strip comments from escaped HTML",
)
# Display data config.
safe_output_keys = Set(
config=True,
default_value={
'metadata', # Not a mimetype per-se, but expected and safe.
'text/plain',
'text/latex',
'application/json',
'image/png',
'image/jpeg',
},
help="Cell output mimetypes to render without modification",
)
sanitized_output_types = Set(
config=True,
default_value={
'text/html',
'text/markdown',
},
help="Cell output types to display after escaping with Bleach.",
)
def preprocess_cell(self, cell, resources, cell_index):
"""
Sanitize potentially-dangerous contents of the cell.
Cell Types:
raw:
Sanitize literal HTML
markdown:
Sanitize literal HTML
code:
Sanitize outputs that could result in code execution
"""
if cell.cell_type == 'raw':
# Sanitize all raw cells anyway.
# Only ones with the text/html mimetype should be emitted
# but erring on the side of safety maybe.
cell.source = self.sanitize_html_tags(cell.source)
return cell, resources
elif cell.cell_type == 'markdown':
cell.source = self.sanitize_html_tags(cell.source)
return cell, resources
elif cell.cell_type == 'code':
cell.outputs = self.sanitize_code_outputs(cell.outputs)
return cell, resources
def sanitize_code_outputs(self, outputs):
"""
Sanitize code cell outputs.
Removes 'text/javascript' fields from display_data outputs, and
runs `sanitize_html_tags` over 'text/html'.
"""
for output in outputs:
# These are always ascii, so nothing to escape.
if output['output_type'] in ('stream', 'error'):
continue
data = output.data
to_remove = []
for key in data:
if key in self.safe_output_keys:
continue
elif key in self.sanitized_output_types:
self.log.info("Sanitizing %s" % key)
data[key] = self.sanitize_html_tags(data[key])
else:
# Mark key for removal. (Python doesn't allow deletion of
# keys from a dict during iteration)
to_remove.append(key)
for key in to_remove:
self.log.info("Removing %s" % key)
del data[key]
return outputs
def sanitize_html_tags(self, html_str):
"""
Sanitize a string containing raw HTML tags.
"""
return clean(
html_str,
tags=self.tags,
attributes=self.attributes,
styles=self.styles,
strip=self.strip,
strip_comments=self.strip_comments,
)
class _Selection(LabeledWidget, ValueWidget, CoreWidget):
"""Base class for Selection widgets
``options`` can be specified as a list of values, list of (label, value)
tuples, or a dict of {label: value}. The labels are the strings that will be
displayed in the UI, representing the actual Python choices, and should be
unique. If labels are not specified, they are generated from the values.
When programmatically setting the value, a reverse lookup is performed
among the options to check that the value is valid. The reverse lookup uses
the equality operator by default, but another predicate may be provided via
the ``equals`` keyword argument. For example, when dealing with numpy arrays,
one may set equals=np.array_equal.
Only labels are synced (values are converted to/from labels), so the labels should
be unique.
"""
value = Any(help="Selected value").tag(sync=True,
to_json=_value_to_label,
from_json=_label_to_value)
options = Union(
[List(), Dict()],
help=
"""List of values, or (label, value) tuples, or a dict of {label: value} pairs that the user can select.
The labels are the strings that will be displayed in the UI, representing the
actual Python choices, and should be unique. If labels are not specified, they
are generated from the values.
The keys are also available as _options_labels.
""")
_options_dict = Dict(read_only=True)
_options_labels = Tuple(read_only=True).tag(sync=True)
_options_values = Tuple(read_only=True)
_model_module = Unicode('jupyter-js-widgets').tag(sync=True)
_view_module = Unicode('jupyter-js-widgets').tag(sync=True)
disabled = Bool(help="Enable or disable user changes").tag(sync=True)
def __init__(self, *args, **kwargs):
self.equals = kwargs.pop('equals', lambda x, y: x == y)
super(_Selection, self).__init__(*args, **kwargs)
def _make_options(self, x):
"""Standardize the options list format.
The returned list should be in the format [('label', value), ('label', value), ...].
The input can be
* a Mapping of labels to values
* an iterable of values (of which at least one is not a list or tuple of length 2)
* an iterable with entries that are lists or tuples of the form ('label', value)
"""
# Return a list of key-value pairs where the keys are strings
# If x is a dict, convert it to list format.
if isinstance(x, Mapping):
return [(unicode_type(k), v) for k, v in x.items()]
# If any entry of x is not a list or tuple of length 2, convert
# the entries to unicode for the labels.
for y in x:
if not (isinstance(y, (list, tuple)) and len(y) == 2):
return [(unicode_type(i), i) for i in x]
# x is already in the correct format: a list of 2-tuples.
# The first element of each tuple should be unicode, this might
# not yet be the case.
return [(unicode_type(k), v) for k, v in x]
@validate('options')
def _validate_options(self, proposal):
"""Handles when the options tuple has been changed.
Setting options with a dict implies setting option labels from the keys of the dict.
"""
new = proposal['value']
options = self._make_options(new)
self.set_trait('_options_dict', dict(options))
self.set_trait('_options_labels', [i[0] for i in options])
self.set_trait('_options_values', [i[1] for i in options])
return new
@observe('options')
def _value_in_options(self, change):
"Ensure the value is an option; if not, set to the first value"
# ensure that the chosen value is still one of the options
if len(self.options) == 0:
self.value = None
else:
try:
_value_to_label(self.value, self)
except KeyError:
self.value = self._options_values[0]
@validate('value')
def _validate_value(self, proposal):
value = proposal['value']
if len(self.options) == 0:
if value is None:
return value
else:
raise TraitError('Invalid selection: empty options list')
else:
try:
_value_to_label(value, self)
return value
except KeyError:
raise TraitError('Invalid selection')
class Kernel(SingletonConfigurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
@observe('eventloop')
def _update_eventloop(self, change):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
session = Instance(Session, allow_none=True)
profile_dir = Instance('IPython.core.profiledir.ProfileDir', allow_none=True)
shell_streams = List()
control_stream = Instance(ZMQStream, allow_none=True)
iopub_socket = Any()
iopub_thread = Any()
stdin_socket = Any()
log = Instance(logging.Logger, allow_none=True)
# identities:
int_id = Integer(-1)
ident = Unicode()
@default('ident')
def _default_ident(self):
return unicode_type(uuid.uuid4())
# This should be overridden by wrapper kernels that implement any real
# language.
language_info = {}
# any links that should go in the help menu
help_links = List()
# Private interface
_darwin_app_nap = Bool(True,
help="""Whether to use appnope for compatiblity with OS X App Nap.
Only affects OS X >= 10.9.
"""
).tag(config=True)
# track associations with current request
_allow_stdin = Bool(False)
_parent_header = Dict()
_parent_ident = Any(b'')
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005).tag(config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05).tag(config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# set of aborted msg_ids
aborted = Set()
# Track execution count here. For IPython, we override this to use the
# execution count we store in the shell.
execution_count = 0
msg_types = [
'execute_request', 'complete_request',
'inspect_request', 'history_request',
'comm_info_request', 'kernel_info_request',
'connect_request', 'shutdown_request',
'is_complete_request',
# deprecated:
'apply_request',
]
# add deprecated ipyparallel control messages
control_msg_types = msg_types + ['clear_request', 'abort_request']
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Build dict of handlers for message types
self.shell_handlers = {}
for msg_type in self.msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
self.control_handlers = {}
for msg_type in self.control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents,msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def should_handle(self, stream, msg, idents):
"""Check whether a shell-channel message should be handled
Allows subclasses to prevent handling of certain messages (e.g. aborted requests).
"""
msg_id = msg['header']['msg_id']
if msg_id in self.aborted:
msg_type = msg['header']['msg_type']
# is it safe to assume a msg_id will not be resubmitted?
self.aborted.remove(msg_id)
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status' : 'aborted'}
md = {'engine' : self.ident}
md.update(status)
self.session.send(stream, reply_type, metadata=md,
content=status, parent=msg, ident=idents)
return False
return True
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents,msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if not self.should_handle(stream, msg, idents):
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
self.log.debug("%s: %s", msg_type, msg)
self.pre_handler_hook()
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
self.post_handler_hook()
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def pre_handler_hook(self):
"""Hook to execute before calling message handler"""
# ensure default_int_handler during handler call
self.saved_sigint_handler = signal(SIGINT, default_int_handler)
def post_handler_hook(self):
"""Hook to execute after calling message handler"""
signal(SIGINT, self.saved_sigint_handler)
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop %s", self.eventloop)
for stream in self.shell_streams:
# flush any pending replies,
# which may be skipped by entering the eventloop
stream.flush(zmq.POLLOUT)
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
# publish idle status
self._publish_status('starting')
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _publish_execute_input(self, code, parent, execution_count):
"""Publish the code request on the iopub stream."""
self.session.send(self.iopub_socket, u'execute_input',
{u'code':code, u'execution_count': execution_count},
parent=parent, ident=self._topic('execute_input')
)
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent or self._parent_header,
ident=self._topic('status'),
)
def set_parent(self, ident, parent):
"""Set the current parent_header
Side effects (IOPub messages) and replies are associated with
the request that caused them via the parent_header.
The parent identity is used to route input_request messages
on the stdin channel.
"""
self._parent_ident = ident
self._parent_header = parent
def send_response(self, stream, msg_or_type, content=None, ident=None,
buffers=None, track=False, header=None, metadata=None):
"""Send a response to the message we're currently processing.
This accepts all the parameters of :meth:`jupyter_client.session.Session.send`
except ``parent``.
This relies on :meth:`set_parent` having been called for the current
message.
"""
return self.session.send(stream, msg_or_type, content, self._parent_header,
ident, buffers, track, header, metadata)
def init_metadata(self, parent):
"""Initialize metadata.
Run at the beginning of execution requests.
"""
return {
'started': datetime.now(),
}
def finish_metadata(self, parent, metadata, reply_content):
"""Finish populating metadata.
Run after completing an execution request.
"""
return metadata
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
try:
content = parent[u'content']
code = py3compat.cast_unicode_py2(content[u'code'])
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
user_expressions = content.get('user_expressions', {})
allow_stdin = content.get('allow_stdin', False)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
stop_on_error = content.get('stop_on_error', True)
metadata = self.init_metadata(parent)
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self.execution_count += 1
self._publish_execute_input(code, parent, self.execution_count)
reply_content = self.do_execute(code, silent, store_history,
user_expressions, allow_stdin)
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
metadata = self.finish_metadata(parent, metadata, reply_content)
reply_msg = self.session.send(stream, u'execute_reply',
reply_content, parent, metadata=metadata,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content']['status'] == u'error' and stop_on_error:
self._abort_queues()
def do_execute(self, code, silent, store_history=True,
user_expressions=None, allow_stdin=False):
"""Execute user code. Must be overridden by subclasses.
"""
raise NotImplementedError
def complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
cursor_pos = content['cursor_pos']
matches = self.do_complete(code, cursor_pos)
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply',
matches, parent, ident)
self.log.debug("%s", completion_msg)
def do_complete(self, code, cursor_pos):
"""Override in subclasses to find completions.
"""
return {'matches' : [],
'cursor_end' : cursor_pos,
'cursor_start' : cursor_pos,
'metadata' : {},
'status' : 'ok'}
def inspect_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_inspect(content['code'], content['cursor_pos'],
content.get('detail_level', 0))
# Before we send this object over, we scrub it for JSON usage
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'inspect_reply',
reply_content, parent, ident)
self.log.debug("%s", msg)
def do_inspect(self, code, cursor_pos, detail_level=0):
"""Override in subclasses to allow introspection.
"""
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
def history_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_history(**content)
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'history_reply',
reply_content, parent, ident)
self.log.debug("%s", msg)
def do_history(self, hist_access_type, output, raw, session=None, start=None,
stop=None, n=None, pattern=None, unique=False):
"""Override in subclasses to access history.
"""
return {'status': 'ok', 'history': []}
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
content['status'] = 'ok'
msg = self.session.send(stream, 'connect_reply',
content, parent, ident)
self.log.debug("%s", msg)
@property
def kernel_info(self):
return {
'protocol_version': kernel_protocol_version,
'implementation': self.implementation,
'implementation_version': self.implementation_version,
'language_info': self.language_info,
'banner': self.banner,
'help_links': self.help_links,
}
def kernel_info_request(self, stream, ident, parent):
content = {'status': 'ok'}
content.update(self.kernel_info)
msg = self.session.send(stream, 'kernel_info_reply',
content, parent, ident)
self.log.debug("%s", msg)
def comm_info_request(self, stream, ident, parent):
content = parent['content']
target_name = content.get('target_name', None)
# Should this be moved to ipkernel?
if hasattr(self, 'comm_manager'):
comms = {
k: dict(target_name=v.target_name)
for (k, v) in self.comm_manager.comms.items()
if v.target_name == target_name or target_name is None
}
else:
comms = {}
reply_content = dict(comms=comms, status='ok')
msg = self.session.send(stream, 'comm_info_reply',
reply_content, parent, ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
content = self.do_shutdown(parent['content']['restart'])
self.session.send(stream, u'shutdown_reply', content, parent, ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply',
content, parent
)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time()+0.1, loop.stop)
def do_shutdown(self, restart):
"""Override in subclasses to do things when the frontend shuts down the
kernel.
"""
return {'status': 'ok', 'restart': restart}
def is_complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
reply_content = self.do_is_complete(code)
reply_content = json_clean(reply_content)
reply_msg = self.session.send(stream, 'is_complete_reply',
reply_content, parent, ident)
self.log.debug("%s", reply_msg)
def do_is_complete(self, code):
"""Override in subclasses to find completions.
"""
return {'status' : 'unknown',
}
#---------------------------------------------------------------------------
# Engine methods (DEPRECATED)
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
self.log.warn("""apply_request is deprecated in kernel_base, moving to ipyparallel.""")
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
md = self.init_metadata(parent)
reply_content, result_buf = self.do_apply(content, bufs, msg_id, md)
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
md = self.finish_metadata(parent, md, reply_content)
self.session.send(stream, u'apply_reply', reply_content,
parent=parent, ident=ident,buffers=result_buf, metadata=md)
def do_apply(self, content, bufs, msg_id, reply_metadata):
"""DEPRECATED"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Control messages (DEPRECATED)
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specific msg by id"""
self.log.warn("abort_request is deprecated in kernel_base. It os only part of IPython parallel")
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, string_types):
msg_ids = [msg_ids]
if not msg_ids:
self._abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream, 'abort_reply', content=content,
parent=parent, ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
self.log.warn("clear_request is deprecated in kernel_base. It os only part of IPython parallel")
content = self.do_clear()
self.session.send(stream, 'clear_reply', ident=idents, parent=parent,
content = content)
def do_clear(self):
"""DEPRECATED"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status' : 'aborted'}
md = {'engine' : self.ident}
md.update(status)
reply_msg = self.session.send(stream, reply_type, metadata=md,
content=status, parent=msg, ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def getpass(self, prompt=''):
"""Forward getpass to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"getpass was called, but this frontend does not support input requests."
)
return self._input_request(prompt,
self._parent_ident,
self._parent_header,
password=True,
)
def raw_input(self, prompt=''):
"""Forward raw_input to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"raw_input was called, but this frontend does not support input requests."
)
return self._input_request(str(prompt),
self._parent_ident,
self._parent_header,
password=False,
)
def _input_request(self, prompt, ident, parent, password=False):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# flush the stdin socket, to purge stale replies
while True:
try:
self.stdin_socket.recv_multipart(zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
break
else:
raise
# Send the input request.
content = json_clean(dict(prompt=prompt, password=password))
self.session.send(self.stdin_socket, u'input_request', content, parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
except KeyboardInterrupt:
# re-raise KeyboardInterrupt, to truncate traceback
raise KeyboardInterrupt
else:
break
try:
value = py3compat.unicode_to_str(reply['content']['value'])
except:
self.log.error("Bad input_reply: %s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp,
ConnectionFileMixin):
name = 'ipython-kernel'
aliases = Dict(kernel_aliases)
flags = Dict(kernel_flags)
classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session]
# the kernel class, as an importstring
kernel_class = Type('ipykernel.ipkernel.IPythonKernel',
klass='ipykernel.kernelbase.Kernel',
help="""The Kernel subclass to be used.
This should allow easy re-use of the IPKernelApp entry point
to configure and launch kernels other than IPython's own.
""").tag(config=True)
kernel = Any()
poller = Any(
) # don't restrict this even though current pollers are all Threads
heartbeat = Instance(Heartbeat, allow_none=True)
context = Any()
shell_socket = Any()
control_socket = Any()
debugpy_socket = Any()
debug_shell_socket = Any()
stdin_socket = Any()
iopub_socket = Any()
iopub_thread = Any()
control_thread = Any()
_ports = Dict()
subcommands = {
'install': ('ipykernel.kernelspec.InstallIPythonKernelSpecApp',
'Install the IPython kernel'),
}
# connection info:
connection_dir = Unicode()
@default('connection_dir')
def _default_connection_dir(self):
return jupyter_runtime_dir()
@property
def abs_connection_file(self):
if os.path.basename(self.connection_file) == self.connection_file:
return os.path.join(self.connection_dir, self.connection_file)
else:
return self.connection_file
# streams, etc.
no_stdout = Bool(
False, help="redirect stdout to the null device").tag(config=True)
no_stderr = Bool(
False, help="redirect stderr to the null device").tag(config=True)
trio_loop = Bool(False, help="Set main event loop.").tag(config=True)
quiet = Bool(
True, help="Only send stdout/stderr to output stream").tag(config=True)
outstream_class = DottedObjectName(
'ipykernel.iostream.OutStream',
help="The importstring for the OutStream factory").tag(config=True)
displayhook_class = DottedObjectName(
'ipykernel.displayhook.ZMQDisplayHook',
help="The importstring for the DisplayHook factory").tag(config=True)
capture_fd_output = Bool(
True,
help=
"""Attempt to capture and forward low-level output, e.g. produced by Extension libraries.
""",
).tag(config=True)
# polling
parent_handle = Integer(
int(os.environ.get('JPY_PARENT_PID') or 0),
help="""kill this process if its parent dies. On Windows, the argument
specifies the HANDLE of the parent process, otherwise it is simply boolean.
""").tag(config=True)
interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0),
help="""ONLY USED ON WINDOWS
Interrupt this process when the parent is signaled.
""").tag(config=True)
def init_crash_handler(self):
sys.excepthook = self.excepthook
def excepthook(self, etype, evalue, tb):
# write uncaught traceback to 'real' stderr, not zmq-forwarder
traceback.print_exception(etype, evalue, tb, file=sys.__stderr__)
def init_poller(self):
if sys.platform == 'win32':
if self.interrupt or self.parent_handle:
self.poller = ParentPollerWindows(self.interrupt,
self.parent_handle)
elif self.parent_handle and self.parent_handle != 1:
# PID 1 (init) is special and will never go away,
# only be reassigned.
# Parent polling doesn't work if ppid == 1 to start with.
self.poller = ParentPollerUnix()
def _try_bind_socket(self, s, port):
iface = '%s://%s' % (self.transport, self.ip)
if self.transport == 'tcp':
if port <= 0:
port = s.bind_to_random_port(iface)
else:
s.bind("tcp://%s:%i" % (self.ip, port))
elif self.transport == 'ipc':
if port <= 0:
port = 1
path = "%s-%i" % (self.ip, port)
while os.path.exists(path):
port = port + 1
path = "%s-%i" % (self.ip, port)
else:
path = "%s-%i" % (self.ip, port)
s.bind("ipc://%s" % path)
return port
def _bind_socket(self, s, port):
try:
win_in_use = errno.WSAEADDRINUSE
except AttributeError:
win_in_use = None
# Try up to 100 times to bind a port when in conflict to avoid
# infinite attempts in bad setups
max_attempts = 1 if port else 100
for attempt in range(max_attempts):
try:
return self._try_bind_socket(s, port)
except zmq.ZMQError as ze:
# Raise if we have any error not related to socket binding
if ze.errno != errno.EADDRINUSE and ze.errno != win_in_use:
raise
if attempt == max_attempts - 1:
raise
def write_connection_file(self):
"""write connection info to JSON file"""
cf = self.abs_connection_file
self.log.debug("Writing connection file: %s", cf)
write_connection_file(cf,
ip=self.ip,
key=self.session.key,
transport=self.transport,
shell_port=self.shell_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
iopub_port=self.iopub_port,
control_port=self.control_port)
def cleanup_connection_file(self):
cf = self.abs_connection_file
self.log.debug("Cleaning up connection file: %s", cf)
try:
os.remove(cf)
except OSError:
pass
self.cleanup_ipc_files()
def init_connection_file(self):
if not self.connection_file:
self.connection_file = "kernel-%s.json" % os.getpid()
try:
self.connection_file = filefind(self.connection_file,
['.', self.connection_dir])
except OSError:
self.log.debug("Connection file not found: %s",
self.connection_file)
# This means I own it, and I'll create it in this directory:
os.makedirs(os.path.dirname(self.abs_connection_file),
mode=0o700,
exist_ok=True)
# Also, I will clean it up:
atexit.register(self.cleanup_connection_file)
return
try:
self.load_connection_file()
except Exception:
self.log.error("Failed to load connection file: %r",
self.connection_file,
exc_info=True)
self.exit(1)
def init_sockets(self):
# Create a context, a session, and the kernel sockets.
self.log.info("Starting the kernel at pid: %i", os.getpid())
assert self.context is None, "init_sockets cannot be called twice!"
self.context = context = zmq.Context()
atexit.register(self.close)
self.shell_socket = context.socket(zmq.ROUTER)
self.shell_socket.linger = 1000
self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port)
self.stdin_socket = context.socket(zmq.ROUTER)
self.stdin_socket.linger = 1000
self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port)
if hasattr(zmq, 'ROUTER_HANDOVER'):
# set router-handover to workaround zeromq reconnect problems
# in certain rare circumstances
# see ipython/ipykernel#270 and zeromq/libzmq#2892
self.shell_socket.router_handover = \
self.stdin_socket.router_handover = 1
self.init_control(context)
self.init_iopub(context)
def init_control(self, context):
self.control_socket = context.socket(zmq.ROUTER)
self.control_socket.linger = 1000
self.control_port = self._bind_socket(self.control_socket,
self.control_port)
self.log.debug("control ROUTER Channel on port: %i" %
self.control_port)
self.debugpy_socket = context.socket(zmq.STREAM)
self.debugpy_socket.linger = 1000
self.debug_shell_socket = context.socket(zmq.DEALER)
self.debug_shell_socket.linger = 1000
if self.shell_socket.getsockopt(zmq.LAST_ENDPOINT):
self.debug_shell_socket.connect(
self.shell_socket.getsockopt(zmq.LAST_ENDPOINT))
if hasattr(zmq, 'ROUTER_HANDOVER'):
# set router-handover to workaround zeromq reconnect problems
# in certain rare circumstances
# see ipython/ipykernel#270 and zeromq/libzmq#2892
self.control_socket.router_handover = 1
self.control_thread = ControlThread(daemon=True)
def init_iopub(self, context):
self.iopub_socket = context.socket(zmq.PUB)
self.iopub_socket.linger = 1000
self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port)
self.configure_tornado_logger()
self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True)
self.iopub_thread.start()
# backward-compat: wrap iopub socket API in background thread
self.iopub_socket = self.iopub_thread.background_socket
def init_heartbeat(self):
"""start the heart beating"""
# heartbeat doesn't share context, because it mustn't be blocked
# by the GIL, which is accessed by libzmq when freeing zero-copy messages
hb_ctx = zmq.Context()
self.heartbeat = Heartbeat(hb_ctx,
(self.transport, self.ip, self.hb_port))
self.hb_port = self.heartbeat.port
self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port)
self.heartbeat.start()
def close(self):
"""Close zmq sockets in an orderly fashion"""
# un-capture IO before we start closing channels
self.reset_io()
self.log.info("Cleaning up sockets")
if self.heartbeat:
self.log.debug("Closing heartbeat channel")
self.heartbeat.context.term()
if self.iopub_thread:
self.log.debug("Closing iopub channel")
self.iopub_thread.stop()
self.iopub_thread.close()
if self.control_thread and self.control_thread.is_alive():
self.log.debug("Closing control thread")
self.control_thread.stop()
self.control_thread.join()
if self.debugpy_socket and not self.debugpy_socket.closed:
self.debugpy_socket.close()
if self.debug_shell_socket and not self.debug_shell_socket.closed:
self.debug_shell_socket.close()
for channel in ('shell', 'control', 'stdin'):
self.log.debug("Closing %s channel", channel)
socket = getattr(self, channel + "_socket", None)
if socket and not socket.closed:
socket.close()
self.log.debug("Terminating zmq context")
self.context.term()
self.log.debug("Terminated zmq context")
def log_connection_info(self):
"""display connection info, and store ports"""
basename = os.path.basename(self.connection_file)
if basename == self.connection_file or \
os.path.dirname(self.connection_file) == self.connection_dir:
# use shortname
tail = basename
else:
tail = self.connection_file
lines = [
"To connect another client to this kernel, use:",
" --existing %s" % tail,
]
# log connection info
# info-level, so often not shown.
# frontends should use the %connect_info magic
# to see the connection info
for line in lines:
self.log.info(line)
# also raw print to the terminal if no parent_handle (`ipython kernel`)
# unless log-level is CRITICAL (--quiet)
if not self.parent_handle and self.log_level < logging.CRITICAL:
print(_ctrl_c_message, file=sys.__stdout__)
for line in lines:
print(line, file=sys.__stdout__)
self._ports = dict(shell=self.shell_port,
iopub=self.iopub_port,
stdin=self.stdin_port,
hb=self.hb_port,
control=self.control_port)
def init_blackhole(self):
"""redirects stdout/stderr to devnull if necessary"""
if self.no_stdout or self.no_stderr:
blackhole = open(os.devnull, 'w')
if self.no_stdout:
sys.stdout = sys.__stdout__ = blackhole
if self.no_stderr:
sys.stderr = sys.__stderr__ = blackhole
def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
if sys.stdout is not None:
sys.stdout.flush()
e_stdout = None if self.quiet else sys.__stdout__
e_stderr = None if self.quiet else sys.__stderr__
if not self.capture_fd_output:
outstream_factory = partial(outstream_factory, watchfd=False)
sys.stdout = outstream_factory(self.session,
self.iopub_thread,
'stdout',
echo=e_stdout)
if sys.stderr is not None:
sys.stderr.flush()
sys.stderr = outstream_factory(self.session,
self.iopub_thread,
"stderr",
echo=e_stderr)
if hasattr(sys.stderr, "_original_stdstream_copy"):
for handler in self.log.handlers:
if isinstance(handler, StreamHandler) and (
handler.stream.buffer.fileno() == 2):
self.log.debug(
"Seeing logger to stderr, rerouting to raw filedescriptor."
)
handler.stream = TextIOWrapper(
FileIO(sys.stderr._original_stdstream_copy, "w"))
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
self.displayhook = displayhook_factory(self.session,
self.iopub_socket)
sys.displayhook = self.displayhook
self.patch_io()
def reset_io(self):
"""restore original io
restores state after init_io
"""
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
sys.displayhook = sys.__displayhook__
def patch_io(self):
"""Patch important libraries that can't handle sys.stdout forwarding"""
try:
import faulthandler
except ImportError:
pass
else:
# Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible
# updates to the upstream API and update accordingly (up-to-date as of Python 3.5):
# https://docs.python.org/3/library/faulthandler.html#faulthandler.enable
# change default file to __stderr__ from forwarded stderr
faulthandler_enable = faulthandler.enable
def enable(file=sys.__stderr__, all_threads=True, **kwargs):
return faulthandler_enable(file=file,
all_threads=all_threads,
**kwargs)
faulthandler.enable = enable
if hasattr(faulthandler, 'register'):
faulthandler_register = faulthandler.register
def register(signum,
file=sys.__stderr__,
all_threads=True,
chain=False,
**kwargs):
return faulthandler_register(signum,
file=file,
all_threads=all_threads,
chain=chain,
**kwargs)
faulthandler.register = register
def init_signal(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
def init_kernel(self):
"""Create the Kernel object itself"""
shell_stream = ZMQStream(self.shell_socket)
control_stream = ZMQStream(self.control_socket,
self.control_thread.io_loop)
debugpy_stream = ZMQStream(self.debugpy_socket,
self.control_thread.io_loop)
self.control_thread.start()
kernel_factory = self.kernel_class.instance
kernel = kernel_factory(
parent=self,
session=self.session,
control_stream=control_stream,
debugpy_stream=debugpy_stream,
debug_shell_socket=self.debug_shell_socket,
shell_stream=shell_stream,
control_thread=self.control_thread,
iopub_thread=self.iopub_thread,
iopub_socket=self.iopub_socket,
stdin_socket=self.stdin_socket,
log=self.log,
profile_dir=self.profile_dir,
user_ns=self.user_ns,
)
kernel.record_ports(
{name + '_port': port
for name, port in self._ports.items()})
self.kernel = kernel
# Allow the displayhook to get the execution count
self.displayhook.get_execution_count = lambda: kernel.execution_count
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
# Register inline backend as default
# this is higher priority than matplotlibrc,
# but lower priority than anything else (mpl.use() for instance).
# This only affects matplotlib >= 1.5
if not os.environ.get('MPLBACKEND'):
os.environ[
'MPLBACKEND'] = 'module://matplotlib_inline.backend_inline'
# Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab`
# to ensure that any exception is printed straight to stderr.
# Normally _showtraceback associates the reply with an execution,
# which means frontends will never draw it, as this exception
# is not associated with any execute request.
shell = self.shell
_showtraceback = shell._showtraceback
try:
# replace error-sending traceback with stderr
def print_tb(etype, evalue, stb):
print("GUI event loop or pylab initialization failed",
file=sys.stderr)
print(shell.InteractiveTB.stb2text(stb), file=sys.stderr)
shell._showtraceback = print_tb
InteractiveShellApp.init_gui_pylab(self)
finally:
shell._showtraceback = _showtraceback
def init_shell(self):
self.shell = getattr(self.kernel, 'shell', None)
if self.shell:
self.shell.configurables.append(self)
def configure_tornado_logger(self):
""" Configure the tornado logging.Logger.
Must set up the tornado logger or else tornado will call
basicConfig for the root logger which makes the root logger
go to the real sys.stderr instead of the capture streams.
This function mimics the setup of logging.basicConfig.
"""
logger = logging.getLogger('tornado')
handler = logging.StreamHandler()
formatter = logging.Formatter(logging.BASIC_FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)
def _init_asyncio_patch(self):
"""set default asyncio policy to be compatible with tornado
Tornado 6 (at least) is not compatible with the default
asyncio implementation on Windows
Pick the older SelectorEventLoopPolicy on Windows
if the known-incompatible default policy is in use.
Support for Proactor via a background thread is available in tornado 6.1,
but it is still preferable to run the Selector in the main thread
instead of the background.
do this as early as possible to make it a low priority and overrideable
ref: https://github.com/tornadoweb/tornado/issues/2608
FIXME: if/when tornado supports the defaults in asyncio without threads,
remove and bump tornado requirement for py38.
Most likely, this will mean a new Python version
where asyncio.ProactorEventLoop supports add_reader and friends.
"""
if sys.platform.startswith("win") and sys.version_info >= (3, 8):
import asyncio
try:
from asyncio import (
WindowsProactorEventLoopPolicy,
WindowsSelectorEventLoopPolicy,
)
except ImportError:
pass
# not affected
else:
if type(asyncio.get_event_loop_policy()
) is WindowsProactorEventLoopPolicy:
# WindowsProactorEventLoopPolicy is not compatible with tornado 6
# fallback to the pre-3.8 default of Selector
asyncio.set_event_loop_policy(
WindowsSelectorEventLoopPolicy())
def init_pdb(self):
"""Replace pdb with IPython's version that is interruptible.
With the non-interruptible version, stopping pdb() locks up the kernel in a
non-recoverable state.
"""
import pdb
from IPython.core import debugger
if hasattr(debugger, "InterruptiblePdb"):
# Only available in newer IPython releases:
debugger.Pdb = debugger.InterruptiblePdb
pdb.Pdb = debugger.Pdb
pdb.set_trace = debugger.set_trace
@catch_config_error
def initialize(self, argv=None):
self._init_asyncio_patch()
super().initialize(argv)
if self.subapp is not None:
return
self.init_pdb()
self.init_blackhole()
self.init_connection_file()
self.init_poller()
self.init_sockets()
self.init_heartbeat()
# writing/displaying connection info must be *after* init_sockets/heartbeat
self.write_connection_file()
# Log connection info after writing connection file, so that the connection
# file is definitely available at the time someone reads the log.
self.log_connection_info()
self.init_io()
try:
self.init_signal()
except Exception:
# Catch exception when initializing signal fails, eg when running the
# kernel on a separate thread
if self.log_level < logging.CRITICAL:
self.log.error("Unable to initialize signal:", exc_info=True)
self.init_kernel()
# shell init steps
self.init_path()
self.init_shell()
if self.shell:
self.init_gui_pylab()
self.init_extensions()
self.init_code()
# flush stdout/stderr, so that anything written to these streams during
# initialization do not get associated with the first execution request
sys.stdout.flush()
sys.stderr.flush()
def start(self):
if self.subapp is not None:
return self.subapp.start()
if self.poller is not None:
self.poller.start()
self.kernel.start()
self.io_loop = ioloop.IOLoop.current()
if self.trio_loop:
from ipykernel.trio_runner import TrioRunner
tr = TrioRunner()
tr.initialize(self.kernel, self.io_loop)
try:
tr.run()
except KeyboardInterrupt:
pass
else:
try:
self.io_loop.start()
except KeyboardInterrupt:
pass
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp,
ConnectionFileMixin):
name = 'ipython-kernel'
aliases = Dict(kernel_aliases)
flags = Dict(kernel_flags)
classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session]
# the kernel class, as an importstring
kernel_class = Type('ipykernel.ipkernel.IPythonKernel',
klass='ipykernel.kernelbase.Kernel',
help="""The Kernel subclass to be used.
This should allow easy re-use of the IPKernelApp entry point
to configure and launch kernels other than IPython's own.
""").tag(config=True)
kernel = Any()
poller = Any(
) # don't restrict this even though current pollers are all Threads
heartbeat = Instance(Heartbeat, allow_none=True)
ports = Dict()
subcommands = {
'install': ('ipykernel.kernelspec.InstallIPythonKernelSpecApp',
'Install the IPython kernel'),
}
# connection info:
connection_dir = Unicode()
@default('connection_dir')
def _default_connection_dir(self):
return jupyter_runtime_dir()
@property
def abs_connection_file(self):
if os.path.basename(self.connection_file) == self.connection_file:
return os.path.join(self.connection_dir, self.connection_file)
else:
return self.connection_file
# streams, etc.
no_stdout = Bool(
False, help="redirect stdout to the null device").tag(config=True)
no_stderr = Bool(
False, help="redirect stderr to the null device").tag(config=True)
quiet = Bool(
True, help="Only send stdout/stderr to output stream").tag(config=True)
outstream_class = DottedObjectName(
'ipykernel.iostream.OutStream',
help="The importstring for the OutStream factory").tag(config=True)
displayhook_class = DottedObjectName(
'ipykernel.displayhook.ZMQDisplayHook',
help="The importstring for the DisplayHook factory").tag(config=True)
# polling
parent_handle = Integer(
int(os.environ.get('JPY_PARENT_PID') or 0),
help="""kill this process if its parent dies. On Windows, the argument
specifies the HANDLE of the parent process, otherwise it is simply boolean.
""").tag(config=True)
interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0),
help="""ONLY USED ON WINDOWS
Interrupt this process when the parent is signaled.
""").tag(config=True)
def init_crash_handler(self):
sys.excepthook = self.excepthook
def excepthook(self, etype, evalue, tb):
# write uncaught traceback to 'real' stderr, not zmq-forwarder
traceback.print_exception(etype, evalue, tb, file=sys.__stderr__)
def init_poller(self):
if sys.platform == 'win32':
if self.interrupt or self.parent_handle:
self.poller = ParentPollerWindows(self.interrupt,
self.parent_handle)
elif self.parent_handle and self.parent_handle != 1:
# PID 1 (init) is special and will never go away,
# only be reassigned.
# Parent polling doesn't work if ppid == 1 to start with.
self.poller = ParentPollerUnix()
def _bind_socket(self, s, port):
iface = '%s://%s' % (self.transport, self.ip)
if self.transport == 'tcp':
if port <= 0:
port = s.bind_to_random_port(iface)
else:
s.bind("tcp://%s:%i" % (self.ip, port))
elif self.transport == 'ipc':
if port <= 0:
port = 1
path = "%s-%i" % (self.ip, port)
while os.path.exists(path):
port = port + 1
path = "%s-%i" % (self.ip, port)
else:
path = "%s-%i" % (self.ip, port)
s.bind("ipc://%s" % path)
return port
def write_connection_file(self):
"""write connection info to JSON file"""
cf = self.abs_connection_file
self.log.debug("Writing connection file: %s", cf)
write_connection_file(cf,
ip=self.ip,
key=self.session.key,
transport=self.transport,
shell_port=self.shell_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
iopub_port=self.iopub_port,
control_port=self.control_port)
def cleanup_connection_file(self):
cf = self.abs_connection_file
self.log.debug("Cleaning up connection file: %s", cf)
try:
os.remove(cf)
except (IOError, OSError):
pass
self.cleanup_ipc_files()
def init_connection_file(self):
if not self.connection_file:
self.connection_file = "kernel-%s.json" % os.getpid()
try:
self.connection_file = filefind(self.connection_file,
['.', self.connection_dir])
except IOError:
self.log.debug("Connection file not found: %s",
self.connection_file)
# This means I own it, and I'll create it in this directory:
ensure_dir_exists(os.path.dirname(self.abs_connection_file), 0o700)
# Also, I will clean it up:
atexit.register(self.cleanup_connection_file)
return
try:
self.load_connection_file()
except Exception:
self.log.error("Failed to load connection file: %r",
self.connection_file,
exc_info=True)
self.exit(1)
def init_sockets(self):
# Create a context, a session, and the kernel sockets.
self.log.info("Starting the kernel at pid: %i", os.getpid())
context = zmq.Context.instance()
# Uncomment this to try closing the context.
# atexit.register(context.term)
self.shell_socket = context.socket(zmq.ROUTER)
self.shell_socket.linger = 1000
self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port)
self.stdin_socket = context.socket(zmq.ROUTER)
self.stdin_socket.linger = 1000
self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port)
self.control_socket = context.socket(zmq.ROUTER)
self.control_socket.linger = 1000
self.control_port = self._bind_socket(self.control_socket,
self.control_port)
self.log.debug("control ROUTER Channel on port: %i" %
self.control_port)
if hasattr(zmq, 'ROUTER_HANDOVER'):
# set router-handover to workaround zeromq reconnect problems
# in certain rare circumstances
# see ipython/ipykernel#270 and zeromq/libzmq#2892
self.shell_socket.router_handover = \
self.control_socket.router_handover = \
self.stdin_socket.router_handover = 1
self.init_iopub(context)
def init_iopub(self, context):
self.iopub_socket = context.socket(zmq.PUB)
self.iopub_socket.linger = 1000
self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port)
self.configure_tornado_logger()
self.iopub_thread = IOPubThread(self.iopub_socket, pipe=True)
self.iopub_thread.start()
# backward-compat: wrap iopub socket API in background thread
self.iopub_socket = self.iopub_thread.background_socket
def init_heartbeat(self):
"""start the heart beating"""
# heartbeat doesn't share context, because it mustn't be blocked
# by the GIL, which is accessed by libzmq when freeing zero-copy messages
hb_ctx = zmq.Context()
self.heartbeat = Heartbeat(hb_ctx,
(self.transport, self.ip, self.hb_port))
self.hb_port = self.heartbeat.port
self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port)
self.heartbeat.start()
def log_connection_info(self):
"""display connection info, and store ports"""
basename = os.path.basename(self.connection_file)
if basename == self.connection_file or \
os.path.dirname(self.connection_file) == self.connection_dir:
# use shortname
tail = basename
else:
tail = self.connection_file
lines = [
"To connect another client to this kernel, use:",
" --existing %s" % tail,
]
# log connection info
# info-level, so often not shown.
# frontends should use the %connect_info magic
# to see the connection info
for line in lines:
self.log.info(line)
# also raw print to the terminal if no parent_handle (`ipython kernel`)
# unless log-level is CRITICAL (--quiet)
if not self.parent_handle and self.log_level < logging.CRITICAL:
io.raw_print(_ctrl_c_message)
for line in lines:
io.raw_print(line)
self.ports = dict(shell=self.shell_port,
iopub=self.iopub_port,
stdin=self.stdin_port,
hb=self.hb_port,
control=self.control_port)
def init_blackhole(self):
"""redirects stdout/stderr to devnull if necessary"""
if self.no_stdout or self.no_stderr:
blackhole = open(os.devnull, 'w')
if self.no_stdout:
sys.stdout = sys.__stdout__ = blackhole
if self.no_stderr:
sys.stderr = sys.__stderr__ = blackhole
def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
if sys.stdout is not None:
sys.stdout.flush()
e_stdout = None if self.quiet else sys.__stdout__
e_stderr = None if self.quiet else sys.__stderr__
sys.stdout = outstream_factory(self.session,
self.iopub_thread,
u'stdout',
echo=e_stdout)
if sys.stderr is not None:
sys.stderr.flush()
sys.stderr = outstream_factory(self.session,
self.iopub_thread,
u'stderr',
echo=e_stderr)
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
self.displayhook = displayhook_factory(self.session,
self.iopub_socket)
sys.displayhook = self.displayhook
self.patch_io()
def patch_io(self):
"""Patch important libraries that can't handle sys.stdout forwarding"""
try:
import faulthandler
except ImportError:
pass
else:
# Warning: this is a monkeypatch of `faulthandler.enable`, watch for possible
# updates to the upstream API and update accordingly (up-to-date as of Python 3.5):
# https://docs.python.org/3/library/faulthandler.html#faulthandler.enable
# change default file to __stderr__ from forwarded stderr
faulthandler_enable = faulthandler.enable
def enable(file=sys.__stderr__, all_threads=True, **kwargs):
return faulthandler_enable(file=file,
all_threads=all_threads,
**kwargs)
faulthandler.enable = enable
if hasattr(faulthandler, 'register'):
faulthandler_register = faulthandler.register
def register(signum,
file=sys.__stderr__,
all_threads=True,
chain=False,
**kwargs):
return faulthandler_register(signum,
file=file,
all_threads=all_threads,
chain=chain,
**kwargs)
faulthandler.register = register
def init_signal(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
def init_kernel(self):
"""Create the Kernel object itself"""
shell_stream = ZMQStream(self.shell_socket)
control_stream = ZMQStream(self.control_socket)
kernel_factory = self.kernel_class.instance
kernel = kernel_factory(
parent=self,
session=self.session,
shell_streams=[shell_stream, control_stream],
iopub_thread=self.iopub_thread,
iopub_socket=self.iopub_socket,
stdin_socket=self.stdin_socket,
log=self.log,
profile_dir=self.profile_dir,
user_ns=self.user_ns,
)
kernel.record_ports(
{name + '_port': port
for name, port in self.ports.items()})
self.kernel = kernel
# Allow the displayhook to get the execution count
self.displayhook.get_execution_count = lambda: kernel.execution_count
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
# Register inline backend as default
# this is higher priority than matplotlibrc,
# but lower priority than anything else (mpl.use() for instance).
# This only affects matplotlib >= 1.5
if not os.environ.get('MPLBACKEND'):
os.environ[
'MPLBACKEND'] = 'module://ipykernel.pylab.backend_inline'
# Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab`
# to ensure that any exception is printed straight to stderr.
# Normally _showtraceback associates the reply with an execution,
# which means frontends will never draw it, as this exception
# is not associated with any execute request.
shell = self.shell
_showtraceback = shell._showtraceback
try:
# replace error-sending traceback with stderr
def print_tb(etype, evalue, stb):
print("GUI event loop or pylab initialization failed",
file=sys.stderr)
print(shell.InteractiveTB.stb2text(stb), file=sys.stderr)
shell._showtraceback = print_tb
InteractiveShellApp.init_gui_pylab(self)
finally:
shell._showtraceback = _showtraceback
def init_shell(self):
self.shell = getattr(self.kernel, 'shell', None)
if self.shell:
self.shell.configurables.append(self)
def init_extensions(self):
super(IPKernelApp, self).init_extensions()
# BEGIN HARDCODED WIDGETS HACK
# Ensure ipywidgets extension is loaded if available
extension_man = self.shell.extension_manager
if 'ipywidgets' not in extension_man.loaded:
try:
extension_man.load_extension('ipywidgets')
except ImportError as e:
self.log.debug(
'ipywidgets package not installed. Widgets will not be available.'
)
# END HARDCODED WIDGETS HACK
def configure_tornado_logger(self):
""" Configure the tornado logging.Logger.
Must set up the tornado logger or else tornado will call
basicConfig for the root logger which makes the root logger
go to the real sys.stderr instead of the capture streams.
This function mimics the setup of logging.basicConfig.
"""
logger = logging.getLogger('tornado')
handler = logging.StreamHandler()
formatter = logging.Formatter(logging.BASIC_FORMAT)
handler.setFormatter(formatter)
logger.addHandler(handler)
@catch_config_error
def initialize(self, argv=None):
super(IPKernelApp, self).initialize(argv)
if self.subapp is not None:
return
# register zmq IOLoop with tornado
zmq_ioloop.install()
self.init_blackhole()
self.init_connection_file()
self.init_poller()
self.init_sockets()
self.init_heartbeat()
# writing/displaying connection info must be *after* init_sockets/heartbeat
self.write_connection_file()
# Log connection info after writing connection file, so that the connection
# file is definitely available at the time someone reads the log.
self.log_connection_info()
self.init_io()
self.init_signal()
self.init_kernel()
# shell init steps
self.init_path()
self.init_shell()
if self.shell:
self.init_gui_pylab()
self.init_extensions()
self.init_code()
# flush stdout/stderr, so that anything written to these streams during
# initialization do not get associated with the first execution request
sys.stdout.flush()
sys.stderr.flush()
def start(self):
if self.subapp is not None:
return self.subapp.start()
if self.poller is not None:
self.poller.start()
self.kernel.start()
self.io_loop = ioloop.IOLoop.current()
try:
self.io_loop.start()
except KeyboardInterrupt:
pass
class QgridWidget(widgets.DOMWidget):
"""
The widget class which is instantiated by the 'show_grid' method, and
can also be constructed directly. All of the parameters listed below
can be read/updated after instantiation via attributes of the same name
as the parameter (since they're implemented as traitlets).
When new values are set for any of these options after instantiation
(such as df, grid_options, etc), the change takes effect immediately by
regenerating the SlickGrid control.
Parameters
----------
df : DataFrame
The DataFrame that will be displayed by this instance of
QgridWidget.
grid_options : dict
Options to use when creating the SlickGrid control (i.e. the
interactive grid). See the Notes section below for more information
on the available options, as well as the default options that this
widget uses.
precision : integer
The number of digits of precision to display for floating-point
values. If unset, we use the value of
`pandas.get_option('display.precision')`.
show_toolbar : bool
Whether to show a toolbar with options for adding/removing rows.
Adding/removing rows is an experimental feature which only works
with DataFrames that have an integer index.
Notes
-----
The following dictionary is used for ``grid_options`` if none are
provided explicitly::
{
'fullWidthRows': True,
'syncColumnCellResize': True,
'forceFitColumns': True,
'defaultColumnWidth': 150,
'rowHeight': 28,
'enableColumnReorder': False,
'enableTextSelectionOnCells': True,
'editable': True,
'autoEdit': False,
'explicitInitialization': True,
'maxVisibleRows': 15,
'minVisibleRows': 8,
'sortable': True,
'filterable': True,
'highlightSelectedCell': False,
'highlightSelectedRow': True
}
Most of these options are SlickGrid options which are described
in the `SlickGrid documentation
<https://github.com/mleibman/SlickGrid/wiki/Grid-Options>`_. The
exceptions are the last 6 options listed, which are options that were
added specifically for Qgrid and therefore are not documented in the
SlickGrid documentation.
The first two, `maxVisibleRows` and `minVisibleRows`, allow you to set
an upper and lower bound on the height of your Qgrid widget in terms of
number of rows that are visible.
The next two, `sortable` and `filterable`, control whether qgrid will
allow the user to sort and filter, respectively. If you set `sortable` to
False nothing will happen when the column headers are clicked.
If you set `filterable` to False, the filter icons won't be shown for any
columns.
The last two, `highlightSelectedCell` and `highlightSelectedRow`, control
how the styling of qgrid changes when a cell is selected. If you set
`highlightSelectedCell` to True, the selected cell will be given
a light blue border. If you set `highlightSelectedRow` to False, the
light blue background that's shown by default for selected rows will be
hidden.
See Also
--------
set_defaults : Permanently set global defaults for the parameters
of the QgridWidget constructor, with the exception of
the ``df`` parameter.
set_grid_option : Permanently set global defaults for individual
SlickGrid options. Does so by changing the default
for the ``grid_options`` parameter of the QgridWidget
constructor.
Attributes
----------
df : DataFrame
Get/set the DataFrame that's being displayed by the current instance.
This DataFrame will NOT reflect any sorting/filtering/editing
changes that are made via the UI. To get a copy of the DataFrame that
does reflect sorting/filtering/editing changes, use the
``get_changed_df()`` method.
grid_options : dict
Get/set the SlickGrid options being used by the current instance.
precision : integer
Get/set the precision options being used by the current instance.
show_toolbar : bool
Get/set the show_toolbar option being used by the current instance.
"""
_view_name = Unicode('QgridView').tag(sync=True)
_model_name = Unicode('QgridModel').tag(sync=True)
_view_module = Unicode('qgrid').tag(sync=True)
_model_module = Unicode('qgrid').tag(sync=True)
_view_module_version = Unicode('1.0.2').tag(sync=True)
_model_module_version = Unicode('1.0.2').tag(sync=True)
_df = Instance(pd.DataFrame)
_df_json = Unicode('', sync=True)
_primary_key = List()
_columns = Dict({}, sync=True)
_filter_tables = Dict({})
_sorted_column_cache = Dict({})
_interval_columns = List([], sync=True)
_period_columns = List([])
_string_columns = List([])
_sort_helper_columns = Dict({})
_initialized = Bool(False)
_ignore_df_changed = Bool(False)
_unfiltered_df = Instance(pd.DataFrame)
_index_col_name = Unicode('qgrid_unfiltered_index', sync=True)
_sort_col_suffix = Unicode('_qgrid_sort_column')
_multi_index = Bool(False)
_edited = Bool(False)
_selected_rows = List([])
_viewport_range = Tuple(Integer(), Integer(), default_value=(0, 100))
_df_range = Tuple(Integer(), Integer(), default_value=(0, 100), sync=True)
_row_count = Integer(0, sync=True)
_sort_field = Any('', sync=True)
_sort_ascending = Bool(True, sync=True)
df = Instance(pd.DataFrame)
precision = Integer(6, sync=True)
grid_options = Dict(sync=True)
show_toolbar = Bool(False, sync=True)
def __init__(self, *args, **kwargs):
self._initialized = False
super(QgridWidget, self).__init__(*args, **kwargs)
# register a callback for custom messages
self.on_msg(self._handle_qgrid_msg)
self._initialized = True
if self.df is not None:
self._update_df()
def _grid_options_default(self):
return defaults.grid_options
def _precision_default(self):
return defaults.precision
def _show_toolbar_default(self):
return defaults.show_toolbar
def _update_df(self):
self._ignore_df_changed = True
# make a copy of the user's dataframe
self._df = self.df.copy()
# insert a column which we'll use later to map edits from
# a filtered version of this df back to the unfiltered version
self._df.insert(0, self._index_col_name, range(0, len(self._df)))
# keep an unfiltered version to serve as the starting point
# for filters, and the state we return to when filters are removed
self._unfiltered_df = self._df.copy()
self._update_table(update_columns=True, fire_data_change_event=False)
self._ignore_df_changed = False
def _rebuild_widget(self):
self._update_df()
self.send({'type': 'draw_table'})
def _df_changed(self):
"""Build the Data Table for the DataFrame."""
if self._ignore_df_changed or not self._initialized:
return
self._rebuild_widget()
def _precision_changed(self):
if not self._initialized:
return
self._rebuild_widget()
def _grid_options_changed(self):
if not self._initialized:
return
self._rebuild_widget()
def _show_toolbar_changed(self):
if not self._initialized:
return
self._rebuild_widget()
def _update_table(self,
update_columns=False,
triggered_by=None,
scroll_to_row=None,
fire_data_change_event=True):
df = self._df.copy()
from_index = max(self._viewport_range[0] - PAGE_SIZE, 0)
to_index = max(self._viewport_range[0] + PAGE_SIZE, 0)
new_df_range = (from_index, to_index)
if triggered_by is 'viewport_changed' and \
self._df_range == new_df_range:
return
self._df_range = new_df_range
df = df.iloc[from_index:to_index]
self._row_count = len(self._df.index)
if type(df.index) == pd.core.index.MultiIndex:
self._multi_index = True
else:
self._multi_index = False
if update_columns:
self._string_columns = list(df.select_dtypes(
include=[np.dtype('O'), 'category']
).columns.values)
# call map(str) for all columns identified as string columns, in
# case any are not strings already
for col_name in self._string_columns:
sort_column_name = self._sort_helper_columns.get(col_name)
if sort_column_name:
series_to_set = df[sort_column_name]
else:
series_to_set = self._get_col_series_from_df(
col_name, df
).map(str)
self._set_col_series_on_df(col_name, df, series_to_set)
df_json = pd_json.to_json(None, df,
orient='table',
date_format='iso',
double_precision=self.precision)
if update_columns:
self._interval_columns = []
self._sort_helper_columns = {}
self._period_columns = []
# parse the schema that we just exported in order to get the
# column metadata that was generated by 'to_json'
parsed_json = json.loads(df_json)
df_schema = parsed_json['schema']
if ('primaryKey' in df_schema):
self._primary_key = df_schema['primaryKey']
else:
# for some reason, 'primaryKey' isn't set when the index is
# a single interval column. that's why this case is here.
self._primary_key = [df.index.name]
columns = {}
for i, cur_column in enumerate(df_schema['fields']):
col_name = cur_column['name']
if 'constraints' in cur_column and \
isinstance(cur_column['constraints']['enum'][0], dict):
cur_column['type'] = 'interval'
self._interval_columns.append(col_name)
if 'freq' in cur_column:
self._period_columns.append(col_name)
if col_name in self._primary_key:
cur_column['is_index'] = True
cur_column['position'] = i
columns[col_name] = cur_column
self._columns = columns
# special handling for interval columns: convert to a string column
# and then call 'to_json' again to get a new version of the table
# json that has interval columns replaced with text columns
if len(self._interval_columns) > 0:
for col_name in self._interval_columns:
col_series = self._get_col_series_from_df(col_name, df)
col_series_as_strings = col_series.map(lambda x: str(x))
self._set_col_series_on_df(col_name, df,
col_series_as_strings)
# special handling for period index columns: call to_timestamp to
# convert the series to a datetime series before displaying
if len(self._period_columns) > 0:
for col_name in self._period_columns:
sort_column_name = self._sort_helper_columns.get(col_name)
if sort_column_name:
series_to_set = df[sort_column_name]
else:
series_to_set = self._get_col_series_from_df(
col_name, df
).to_timestamp()
self._set_col_series_on_df(col_name, df, series_to_set)
# and then call 'to_json' again to get a new version of the table
# json that has interval columns replaced with text columns
if len(self._interval_columns) > 0 or len(self._period_columns) > 0:
df_json = pd_json.to_json(None, df,
orient='table',
date_format='iso',
double_precision=self.precision)
self._df_json = df_json
if fire_data_change_event:
data_to_send = {
'type': 'update_data_view',
'columns': self._columns,
'triggered_by': triggered_by
}
if scroll_to_row:
data_to_send['scroll_to_row'] = scroll_to_row
self.send(data_to_send)
def _update_sort(self):
try:
if self._sort_field == '':
return
if self._sort_field in self._primary_key:
if len(self._primary_key) == 1:
self._df.sort_index(
ascending=self._sort_ascending,
inplace=True
)
else:
level_id = self._sort_field
if self._sort_field.startswith('level_'):
level_id = int(self._sort_field[6:])
self._df.sort_index(
level=level_id,
ascending=self._sort_ascending,
inplace=True
)
else:
self._df.sort_values(
self._sort_field,
ascending=self._sort_ascending,
inplace=True
)
except TypeError:
self.log.info('TypeError occurred, assuming mixed data type '
'column')
# if there's a TypeError, assume it means that we have a mixed
# type column, and attempt to create a stringified version of
# the column to use for sorting/filtering
self._df.sort_values(
self._initialize_sort_column(self._sort_field),
ascending=self._sort_ascending,
inplace=True
)
# Add a new column which is a stringified version of the column whose name
# was passed in, which can be used for sorting and filtering (to avoid
# error caused by the type of data in the column, like having multiple
# data types in a single column).
def _initialize_sort_column(self, col_name, to_timestamp=False):
sort_column_name = self._sort_helper_columns.get(col_name)
if sort_column_name:
return sort_column_name
sort_col_series = \
self._get_col_series_from_df(col_name, self._df)
sort_col_series_unfiltered = \
self._get_col_series_from_df(col_name, self._unfiltered_df)
sort_column_name = str(col_name) + self._sort_col_suffix
if to_timestamp:
self._df[sort_column_name] = sort_col_series.to_timestamp()
self._unfiltered_df[sort_column_name] = \
sort_col_series_unfiltered.to_timestamp()
else:
self._df[sort_column_name] = sort_col_series.map(str)
self._unfiltered_df[sort_column_name] = \
sort_col_series_unfiltered.map(str)
self._sort_helper_columns[col_name] = sort_column_name
return sort_column_name
def _handle_get_column_min_max(self, content):
col_name = content['field']
col_info = self._columns[col_name]
if 'filter_info' in col_info and 'selected' in col_info['filter_info']:
df_for_unique = self._unfiltered_df
else:
df_for_unique = self._df
# if there's a period index column, add a sort column which has the
# same values, but converted to timestamps instead of period objects.
# we'll use that sort column for all subsequent sorts/filters.
if col_name in self._period_columns:
self._initialize_sort_column(col_name,
to_timestamp=True)
col_series = self._get_col_series_from_df(col_name, df_for_unique)
if 'is_index' in col_info:
col_series = pd.Series(col_series)
if col_info['type'] in ['integer', 'number']:
if 'filter_info' not in col_info or \
(col_info['filter_info']['min'] is None and
col_info['filter_info']['max'] is None):
col_info['slider_max'] = max(col_series)
col_info['slider_min'] = min(col_series)
self._columns[col_name] = col_info
self.send({
'type': 'column_min_max_updated',
'field': col_name,
'col_info': col_info
})
return
elif col_info['type'] == 'datetime':
if 'filter_info' not in col_info or \
(col_info['filter_info']['min'] is None and
col_info['filter_info']['max'] is None):
col_info['filter_max'] = max(col_series)
col_info['filter_min'] = min(col_series)
self._columns[col_name] = col_info
self.send({
'type': 'column_min_max_updated',
'field': col_name,
'col_info': col_info
})
return
elif col_info['type'] == 'boolean':
self.log.info('handling boolean type')
if 'filter_info' not in col_info:
values = []
for possible_val in [True, False]:
if possible_val in col_series:
values.append(possible_val)
col_info['values'] = values
self._columns[col_name] = col_info
self.send({
'type': 'column_min_max_updated',
'field': col_name,
'col_info': col_info
})
self.log.info('handled boolean type')
return
else:
if col_info['type'] == 'any':
unique_list = col_series.dtype.categories
else:
if col_name in self._sorted_column_cache:
unique_list = self._sorted_column_cache[col_name]
else:
unique = col_series.unique()
if len(unique) < 500000:
try:
unique.sort()
except TypeError:
sort_col_name = \
self._initialize_sort_column(col_name)
col_series = df_for_unique[sort_col_name]
unique = col_series.unique()
unique.sort()
unique_list = unique.tolist()
self._sorted_column_cache[col_name] = unique_list
if content['search_val'] is not None:
unique_list = [
k for k in unique_list if
content['search_val'].lower() in str(k).lower()
]
# if the filter that we're opening is already active (as indicated
# by the presence of a 'selected' attribute on the column's
# filter_info attribute), show the selected rows at the top and
# specify that they should be checked
if 'filter_info' in col_info and \
'selected' in col_info['filter_info']:
col_filter_info = col_info['filter_info']
col_filter_table = self._filter_tables[col_name]
def get_value_from_filter_table(k):
return col_filter_table[k]
selected_indices = col_filter_info['selected'] or []
if selected_indices == 'all':
excluded_indices = col_filter_info['excluded'] or []
excluded_values = list(map(get_value_from_filter_table,
excluded_indices))
non_excluded_count = 0
for i in range(len(unique_list), 0, -1):
unique_val = unique_list[i-1]
if unique_val not in excluded_values:
non_excluded_count += 1
excluded_values.insert(0, unique_val)
col_info['values'] = excluded_values
col_info['selected_length'] = non_excluded_count
elif len(selected_indices) == 0:
col_info['selected_length'] = 0
col_info['values'] = unique_list
else:
selected_vals = list(map(get_value_from_filter_table,
selected_indices))
col_info['selected_length'] = len(selected_vals)
in_selected = set(selected_vals)
in_unique = set(unique_list)
in_unique_but_not_selected = list(in_unique - in_selected)
in_unique_but_not_selected.sort()
selected_vals.extend(in_unique_but_not_selected)
col_info['values'] = selected_vals
else:
col_info['selected_length'] = 0
col_info['values'] = unique_list
length = len(col_info['values'])
self._filter_tables[col_name] = list(col_info['values'])
if col_info['type'] == 'any':
col_info['value_range'] = (0, length)
else:
max_items = PAGE_SIZE * 2
range_max = length
if length > max_items:
col_info['values'] = col_info['values'][:max_items]
range_max = max_items
col_info['value_range'] = (0, range_max)
col_info['length'] = length
self._columns[col_name] = col_info
if content['search_val'] is not None:
message_type = 'update_data_view_filter'
else:
message_type = 'column_min_max_updated'
try:
self.send({
'type': message_type,
'field': col_name,
'col_info': col_info
})
except ValueError:
# if there's a ValueError, assume it's because we're
# attempting to serialize something that can't be converted
# to json, so convert all the values to strings.
col_info['values'] = map(str, col_info['values'])
self.send({
'type': message_type,
'field': col_name,
'col_info': col_info
})
# get any column from a dataframe, including index columns
def _get_col_series_from_df(self, col_name, df):
sort_column_name = self._sort_helper_columns.get(col_name)
if sort_column_name:
return df[sort_column_name]
if col_name in self._primary_key:
if len(self._primary_key) > 1:
key_index = self._primary_key.index(col_name)
return df.index.get_level_values(level=key_index)
else:
return df.index
else:
return df[col_name]
def _set_col_series_on_df(self, col_name, df, col_series):
if col_name in self._primary_key:
col_series.name = col_name
if len(self._primary_key) > 1:
key_index = self._primary_key.index(col_name)
df.index.set_levels(col_series, level=key_index, inplace=True)
else:
df.set_index(col_series, inplace=True)
else:
df[col_name] = col_series
def _append_condition_for_column(self, col_name, filter_info, conditions):
col_series = self._get_col_series_from_df(col_name,
self._unfiltered_df)
if filter_info['type'] == 'slider':
if filter_info['min'] is not None:
conditions.append(col_series >= filter_info['min'])
if filter_info['max'] is not None:
conditions.append(col_series <= filter_info['max'])
elif filter_info['type'] == 'date':
if filter_info['min'] is not None:
conditions.append(
col_series >= pd.to_datetime(filter_info['min'], unit='ms')
)
if filter_info['max'] is not None:
conditions.append(
col_series <= pd.to_datetime(filter_info['max'], unit='ms')
)
elif filter_info['type'] == 'boolean':
if filter_info['selected'] is not None:
conditions.append(
col_series == filter_info['selected']
)
elif filter_info['type'] == 'text':
if col_name not in self._filter_tables:
return
col_filter_table = self._filter_tables[col_name]
selected_indices = filter_info['selected']
excluded_indices = filter_info['excluded']
def get_value_from_filter_table(i):
return col_filter_table[i]
if selected_indices == "all":
if excluded_indices is not None and len(excluded_indices) > 0:
excluded_values = list(
map(get_value_from_filter_table, excluded_indices)
)
conditions.append(~col_series.isin(excluded_values))
elif selected_indices is not None and len(selected_indices) > 0:
selected_values = list(
map(get_value_from_filter_table, selected_indices)
)
conditions.append(col_series.isin(selected_values))
def _handle_filter_changed(self, content):
col_name = content['field']
columns = self._columns.copy()
col_info = columns[col_name]
col_info['filter_info'] = content['filter_info']
columns[col_name] = col_info
conditions = []
for key, value in columns.items():
if 'filter_info' in value:
self._append_condition_for_column(
key, value['filter_info'], conditions
)
self._columns = columns
self._ignore_df_changed = True
if len(conditions) == 0:
self._df = self._unfiltered_df.copy()
else:
combined_condition = conditions[0]
for c in conditions[1:]:
combined_condition = combined_condition & c
self._df = self._unfiltered_df[combined_condition].copy()
if len(self._df) < self._viewport_range[0]:
viewport_size = self._viewport_range[1] - self._viewport_range[0]
range_top = max(0, len(self._df) - viewport_size)
self._viewport_range = (range_top, range_top + viewport_size)
self._sorted_column_cache = {}
self._update_sort()
self._update_table(triggered_by='filter_changed')
self._ignore_df_changed = False
def _handle_qgrid_msg(self, widget, content, buffers=None):
try:
self._handle_qgrid_msg_helper(content)
except Exception as e:
self.log.error(e)
self.log.exception("Unhandled exception while handling msg")
def _handle_qgrid_msg_helper(self, content):
"""Handle incoming messages from the QGridView"""
if 'type' not in content:
return
if content['type'] == 'cell_change':
col_info = self._columns[content['column']]
try:
location = (self._df.index[content['row_index']],
content['column'])
val_to_set = content['value']
if col_info['type'] == 'datetime':
val_to_set = pd.to_datetime(val_to_set)
self._df.at[location] = val_to_set
query = self._unfiltered_df[self._index_col_name] == \
content['unfiltered_index']
self._unfiltered_df.loc[query, content['column']] = val_to_set
self._trigger_df_change_event(location)
except (ValueError, TypeError):
msg = "Error occurred while attempting to edit the " \
"DataFrame. Check the notebook server logs for more " \
"information."
self.log.exception(msg)
self.send({
'type': 'show_error',
'error_msg': msg,
'triggered_by': 'add_row'
})
return
elif content['type'] == 'selection_change':
self._selected_rows = content['rows']
elif content['type'] == 'viewport_changed':
self._viewport_range = (content['top'], content['bottom'])
self._update_table(triggered_by='viewport_changed')
elif content['type'] == 'add_row':
self.add_row()
elif content['type'] == 'remove_row':
self.remove_row()
elif content['type'] == 'viewport_changed_filter':
col_name = content['field']
col_info = self._columns[col_name]
col_filter_table = self._filter_tables[col_name]
from_index = max(content['top'] - PAGE_SIZE, 0)
to_index = max(content['top'] + PAGE_SIZE, 0)
col_info['values'] = col_filter_table[from_index:to_index]
col_info['value_range'] = (from_index, to_index)
self._columns[col_name] = col_info
self.send({
'type': 'update_data_view_filter',
'field': col_name,
'col_info': col_info
})
elif content['type'] == 'sort_changed':
self._sort_field = content['sort_field']
self._sort_ascending = content['sort_ascending']
self._sorted_column_cache = {}
self._update_sort()
self._update_table(triggered_by='sort_changed')
self._trigger_df_change_event()
elif content['type'] == 'get_column_min_max':
self._handle_get_column_min_max(content)
elif content['type'] == 'filter_changed':
self._handle_filter_changed(content)
def _trigger_df_change_event(self, location=None):
self.notify_change(Bunch(
name='_df',
old=None,
new=self._df,
owner=self,
type='change',
location=location,
))
def get_changed_df(self):
"""
Get a copy of the DataFrame that was used to create the current
instance of QgridWidget which reflects the current state of the UI.
This includes any sorting or filtering changes, as well as edits
that have been made by double clicking cells.
:rtype: DataFrame
"""
col_names_to_drop = list(self._sort_helper_columns.values())
col_names_to_drop.append(self._index_col_name)
return self._df.drop(col_names_to_drop, axis=1)
def get_selected_df(self):
"""
Get a DataFrame which reflects the current state of the UI and only
includes the currently selected row(s). Internally it calls
``get_changed_df()`` and then filters down to the selected rows
using ``iloc``.
:rtype: DataFrame
"""
changed_df = self.get_changed_df()
return changed_df.iloc[self._selected_rows]
def get_selected_rows(self):
"""
Get the currently selected rows.
:rtype: List of integers
"""
return self._selected_rows
def add_row(self):
"""
Append a row at the end of the dataframe by duplicating the
last row and incrementing it's index by 1. The feature is only
available for DataFrames that have an integer index.
"""
df = self._df
if not df.index.is_integer():
msg = "Cannot add a row to a table with a non-integer index"
self.send({
'type': 'show_error',
'error_msg': msg,
'triggered_by': 'add_row'
})
return
last_index = max(df.index)
last = df.loc[last_index].copy()
last.name += 1
last[self._index_col_name] = last.name
df.loc[last.name] = last.values
self._unfiltered_df.loc[last.name] = last.values
self._update_table(triggered_by='add_row',
scroll_to_row=df.index.get_loc(last.name))
self._trigger_df_change_event()
def remove_row(self):
"""
Remove the current row from the table.
"""
if self._multi_index:
msg = "Cannot remove a row from a table with a multi index"
self.send({
'type': 'show_error',
'error_msg': msg,
'triggered_by': 'remove_row'
})
return
selected_names = \
map(lambda x: self._df.iloc[x].name, self._selected_rows)
self._df.drop(selected_names, inplace=True)
self._unfiltered_df.drop(selected_names, inplace=True)
self._selected_rows = []
self._update_table(triggered_by='remove_row')
self._trigger_df_change_event()
class SingleUserNotebookApp(LabApp):
"""A Subclass of the regular LabApp that is aware of the parent multiuser context."""
description = dedent("""
Single-user server for JupyterHub. Extends the Jupyter Notebook server.
Meant to be invoked by JupyterHub Spawners, and not directly.
""")
examples = ""
subcommands = {}
version = __version__
classes = LabApp.classes + [HubOAuth]
# disable single-user app's localhost checking
allow_remote_access = True
# don't store cookie secrets
cookie_secret_file = ''
# always generate a new cookie secret on launch
# ensures that each spawn clears any cookies from previous session,
# triggering OAuth again
cookie_secret = Bytes()
def _cookie_secret_default(self):
return os.urandom(32)
user = CUnicode().tag(config=True)
group = CUnicode().tag(config=True)
@default('user')
def _default_user(self):
return os.environ.get('JUPYTERHUB_USER') or ''
@default('group')
def _default_group(self):
return os.environ.get('JUPYTERHUB_GROUP') or ''
@observe('user')
def _user_changed(self, change):
self.log.name = change.new
hub_host = Unicode().tag(config=True)
hub_prefix = Unicode('/hub/').tag(config=True)
@default('keyfile')
def _keyfile_default(self):
return os.environ.get('JUPYTERHUB_SSL_KEYFILE') or ''
@default('certfile')
def _certfile_default(self):
return os.environ.get('JUPYTERHUB_SSL_CERTFILE') or ''
@default('client_ca')
def _client_ca_default(self):
return os.environ.get('JUPYTERHUB_SSL_CLIENT_CA') or ''
@default('hub_prefix')
def _hub_prefix_default(self):
base_url = os.environ.get('JUPYTERHUB_BASE_URL') or '/'
return base_url + 'hub/'
hub_api_url = Unicode().tag(config=True)
@default('hub_api_url')
def _hub_api_url_default(self):
return os.environ.get(
'JUPYTERHUB_API_URL') or 'http://127.0.0.1:8081/hub/api'
# defaults for some configurables that may come from service env variables:
@default('base_url')
def _base_url_default(self):
return os.environ.get('JUPYTERHUB_SERVICE_PREFIX') or '/'
# Note: this may be removed if notebook module is >= 5.0.0b1
@validate('base_url')
def _validate_base_url(self, proposal):
"""ensure base_url starts and ends with /"""
value = proposal.value
if not value.startswith('/'):
value = '/' + value
if not value.endswith('/'):
value = value + '/'
return value
@default('port')
def _port_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
if url.port:
return url.port
elif url.scheme == 'http':
return 80
elif url.scheme == 'https':
return 443
return 8888
@default('ip')
def _ip_default(self):
if os.environ.get('JUPYTERHUB_SERVICE_URL'):
url = urlparse(os.environ['JUPYTERHUB_SERVICE_URL'])
if url.hostname:
return url.hostname
return '127.0.0.1'
aliases = aliases
flags = flags
# disble some single-user configurables
token = ''
open_browser = False
quit_button = False
trust_xheaders = True
login_handler_class = JupyterHubLoginHandler
logout_handler_class = JupyterHubLogoutHandler
port_retries = (
0 # disable port-retries, since the Spawner will tell us what port to use
)
disable_user_config = Bool(
False,
help="""Disable user configuration of single-user server.
Prevents user-writable files that normally configure the single-user server
from being loaded, ensuring admins have full control of configuration.
""",
).tag(config=True)
@validate('notebook_dir')
def _notebook_dir_validate(self, proposal):
value = os.path.expanduser(proposal['value'])
# Strip any trailing slashes
# *except* if it's root
_, path = os.path.splitdrive(value)
if path == os.sep:
return value
value = value.rstrip(os.sep)
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("No such notebook dir: %r" % value)
return value
@default('log_datefmt')
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
@default('log_format')
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
def _confirm_exit(self):
# disable the exit confirmation for background notebook processes
self.io_loop.add_callback_from_signal(self.io_loop.stop)
def migrate_config(self):
if self.disable_user_config:
# disable config-migration when user config is disabled
return
else:
super(SingleUserNotebookApp, self).migrate_config()
@property
def config_file_paths(self):
path = super(SingleUserNotebookApp, self).config_file_paths
if self.disable_user_config:
# filter out user-writable config dirs if user config is disabled
path = list(_exclude_home(path))
return path
@property
def nbextensions_path(self):
path = super(SingleUserNotebookApp, self).nbextensions_path
if self.disable_user_config:
path = list(_exclude_home(path))
return path
@validate('static_custom_path')
def _validate_static_custom_path(self, proposal):
path = proposal['value']
if self.disable_user_config:
path = list(_exclude_home(path))
return path
# create dynamic default http client,
# configured with any relevant ssl config
hub_http_client = Any()
@default('hub_http_client')
def _default_client(self):
ssl_context = make_ssl_context(self.keyfile,
self.certfile,
cafile=self.client_ca)
AsyncHTTPClient.configure(None,
defaults={"ssl_options": ssl_context})
return AsyncHTTPClient()
async def check_hub_version(self):
"""Test a connection to my Hub
- exit if I can't connect at all
- check version and warn on sufficient mismatch
"""
client = self.hub_http_client
RETRIES = 5
for i in range(1, RETRIES + 1):
try:
resp = await client.fetch(self.hub_api_url)
except Exception:
self.log.exception(
"Failed to connect to my Hub at %s (attempt %i/%i). Is it running?",
self.hub_api_url,
i,
RETRIES,
)
await gen.sleep(min(2**i, 16))
else:
break
else:
self.exit(1)
hub_version = resp.headers.get('X-JupyterHub-Version')
_check_version(hub_version, __version__, self.log)
server_name = Unicode()
@default('server_name')
def _server_name_default(self):
return os.environ.get('JUPYTERHUB_SERVER_NAME', '')
hub_activity_url = Unicode(
config=True, help="URL for sending JupyterHub activity updates")
@default('hub_activity_url')
def _default_activity_url(self):
return os.environ.get('JUPYTERHUB_ACTIVITY_URL', '')
hub_activity_interval = Integer(
300,
config=True,
help="""
Interval (in seconds) on which to update the Hub
with our latest activity.
""",
)
@default('hub_activity_interval')
def _default_activity_interval(self):
env_value = os.environ.get('JUPYTERHUB_ACTIVITY_INTERVAL')
if env_value:
return int(env_value)
else:
return 300
_last_activity_sent = Any(allow_none=True)
async def notify_activity(self):
"""Notify jupyterhub of activity"""
client = self.hub_http_client
last_activity = self.web_app.last_activity()
if not last_activity:
self.log.debug("No activity to send to the Hub")
return
if last_activity:
# protect against mixed timezone comparisons
if not last_activity.tzinfo:
# assume naive timestamps are utc
self.log.warning("last activity is using naive timestamps")
last_activity = last_activity.replace(tzinfo=timezone.utc)
if self._last_activity_sent and last_activity < self._last_activity_sent:
self.log.debug("No activity since %s",
self._last_activity_sent)
return
last_activity_timestamp = isoformat(last_activity)
async def notify():
self.log.debug("Notifying Hub of activity %s",
last_activity_timestamp)
req = HTTPRequest(
url=self.hub_activity_url,
method='POST',
headers={
"Authorization":
"token {}".format(self.hub_auth.api_token),
"Content-Type": "application/json",
},
body=json.dumps({
'servers': {
self.server_name: {
'last_activity': last_activity_timestamp
}
},
'last_activity': last_activity_timestamp,
}),
)
try:
await client.fetch(req)
except Exception:
self.log.exception("Error notifying Hub of activity")
return False
else:
return True
await exponential_backoff(
notify,
fail_message="Failed to notify Hub of activity",
start_wait=1,
max_wait=15,
timeout=60,
)
self._last_activity_sent = last_activity
async def keep_activity_updated(self):
if not self.hub_activity_url or not self.hub_activity_interval:
self.log.warning("Activity events disabled")
return
self.log.info("Updating Hub with activity every %s seconds",
self.hub_activity_interval)
while True:
try:
await self.notify_activity()
except Exception as e:
self.log.exception("Error notifying Hub of activity")
# add 20% jitter to the interval to avoid alignment
# of lots of requests from user servers
t = self.hub_activity_interval * (1 + 0.2 *
(random.random() - 0.5))
await asyncio.sleep(t)
def initialize(self, argv=None):
# disable trash by default
# this can be re-enabled by config
self.config.FileContentsManager.delete_to_trash = False
return super().initialize(argv)
def start(self):
self.log.info("Starting jupyterhub-singleuser server version %s",
__version__)
# start by hitting Hub to check version
ioloop.IOLoop.current().run_sync(self.check_hub_version)
ioloop.IOLoop.current().add_callback(self.keep_activity_updated)
super(SingleUserNotebookApp, self).start()
def init_hub_auth(self):
api_token = None
if os.getenv('JPY_API_TOKEN'):
# Deprecated env variable (as of 0.7.2)
api_token = os.environ['JPY_API_TOKEN']
if os.getenv('JUPYTERHUB_API_TOKEN'):
api_token = os.environ['JUPYTERHUB_API_TOKEN']
if not api_token:
self.exit(
"JUPYTERHUB_API_TOKEN env is required to run jupyterhub-singleuser. Did you launch it manually?"
)
self.hub_auth = HubOAuth(
parent=self,
api_token=api_token,
api_url=self.hub_api_url,
hub_prefix=self.hub_prefix,
base_url=self.base_url,
keyfile=self.keyfile,
certfile=self.certfile,
client_ca=self.client_ca,
)
# smoke check
if not self.hub_auth.oauth_client_id:
raise ValueError("Missing OAuth client ID")
def init_webapp(self):
# load the hub-related settings into the tornado settings dict
self.init_hub_auth()
s = self.tornado_settings
s['log_function'] = log_request
s['user'] = self.user
s['group'] = self.group
s['hub_prefix'] = self.hub_prefix
s['hub_host'] = self.hub_host
s['hub_auth'] = self.hub_auth
csp_report_uri = s[
'csp_report_uri'] = self.hub_host + url_path_join(
self.hub_prefix, 'security/csp-report')
headers = s.setdefault('headers', {})
headers['X-JupyterHub-Version'] = __version__
# set CSP header directly to workaround bugs in jupyter/notebook 5.0
headers.setdefault(
'Content-Security-Policy',
';'.join(
["frame-ancestors 'self'",
"report-uri " + csp_report_uri]),
)
super(SingleUserNotebookApp, self).init_webapp()
# add OAuth callback
self.web_app.add_handlers(
r".*$",
[(urlparse(self.hub_auth.oauth_redirect_uri).path,
OAuthCallbackHandler)],
)
# apply X-JupyterHub-Version to *all* request handlers (even redirects)
self.patch_default_headers()
self.patch_templates()
def patch_default_headers(self):
if hasattr(RequestHandler, '_orig_set_default_headers'):
return
RequestHandler._orig_set_default_headers = RequestHandler.set_default_headers
def set_jupyterhub_header(self):
self._orig_set_default_headers()
self.set_header('X-JupyterHub-Version', __version__)
RequestHandler.set_default_headers = set_jupyterhub_header
def patch_templates(self):
"""Patch page templates to add Hub-related buttons"""
self.jinja_template_vars[
'logo_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'logo')
self.jinja_template_vars['hub_host'] = self.hub_host
self.jinja_template_vars['hub_prefix'] = self.hub_prefix
env = self.web_app.settings['jinja2_env']
env.globals[
'hub_control_panel_url'] = self.hub_host + url_path_join(
self.hub_prefix, 'home')
# patch jinja env loading to modify page template
def get_page(name):
if name == 'page.html':
return page_template
orig_loader = env.loader
env.loader = ChoiceLoader([FunctionLoader(get_page), orig_loader])
class MultiKernelManager(LoggingConfigurable):
"""A class for managing multiple kernels."""
default_kernel_name = Unicode(
NATIVE_KERNEL_NAME,
config=True,
help="The name of the default kernel to start")
kernel_spec_manager = Instance(KernelSpecManager, allow_none=True)
kernel_manager_class = DottedObjectName(
"jupyter_client.ioloop.IOLoopKernelManager",
config=True,
help="""The kernel manager class. This is configurable to allow
subclassing of the KernelManager for customized behavior.
""")
def __init__(self, *args, **kwargs):
super(MultiKernelManager, self).__init__(*args, **kwargs)
# Cache all the currently used ports
self.currently_used_ports = set()
@observe('kernel_manager_class')
def _kernel_manager_class_changed(self, change):
self.kernel_manager_factory = self._create_kernel_manager_factory()
kernel_manager_factory = Any(
help="this is kernel_manager_class after import")
@default('kernel_manager_factory')
def _kernel_manager_factory_default(self):
return self._create_kernel_manager_factory()
def _create_kernel_manager_factory(self):
kernel_manager_ctor = import_item(self.kernel_manager_class)
def create_kernel_manager(*args, **kwargs):
if self.shared_context:
if self.context.closed:
# recreate context if closed
self.context = self._context_default()
kwargs.setdefault("context", self.context)
km = kernel_manager_ctor(*args, **kwargs)
if km.cache_ports:
km.shell_port = self._find_available_port(km.ip)
km.iopub_port = self._find_available_port(km.ip)
km.stdin_port = self._find_available_port(km.ip)
km.hb_port = self._find_available_port(km.ip)
km.control_port = self._find_available_port(km.ip)
return km
return create_kernel_manager
def _find_available_port(self, ip):
while True:
tmp_sock = socket.socket()
tmp_sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8)
tmp_sock.bind((ip, 0))
port = tmp_sock.getsockname()[1]
tmp_sock.close()
# This is a workaround for https://github.com/jupyter/jupyter_client/issues/487
# We prevent two kernels to have the same ports.
if port not in self.currently_used_ports:
self.currently_used_ports.add(port)
return port
shared_context = Bool(
True,
config=True,
help="Share a single zmq.Context to talk to all my kernels",
)
_created_context = Bool(False)
context = Instance('zmq.Context')
@default("context")
def _context_default(self):
self._created_context = True
return zmq.Context()
def __del__(self):
if self._created_context and self.context and not self.context.closed:
if self.log:
self.log.debug("Destroying zmq context for %s", self)
self.context.destroy()
try:
super_del = super().__del__
except AttributeError:
pass
else:
super_del()
connection_dir = Unicode('')
_kernels = Dict()
def list_kernel_ids(self):
"""Return a list of the kernel ids of the active kernels."""
# Create a copy so we can iterate over kernels in operations
# that delete keys.
return list(self._kernels.keys())
def __len__(self):
"""Return the number of running kernels."""
return len(self.list_kernel_ids())
def __contains__(self, kernel_id):
return kernel_id in self._kernels
def pre_start_kernel(self, kernel_name, kwargs):
# kwargs should be mutable, passing it as a dict argument.
kernel_id = kwargs.pop('kernel_id', self.new_kernel_id(**kwargs))
if kernel_id in self:
raise DuplicateKernelError('Kernel already exists: %s' % kernel_id)
if kernel_name is None:
kernel_name = self.default_kernel_name
# kernel_manager_factory is the constructor for the KernelManager
# subclass we are using. It can be configured as any Configurable,
# including things like its transport and ip.
constructor_kwargs = {}
if self.kernel_spec_manager:
constructor_kwargs[
'kernel_spec_manager'] = self.kernel_spec_manager
km = self.kernel_manager_factory(connection_file=os.path.join(
self.connection_dir, "kernel-%s.json" % kernel_id),
parent=self,
log=self.log,
kernel_name=kernel_name,
**constructor_kwargs)
return km, kernel_name, kernel_id
def start_kernel(self, kernel_name=None, **kwargs):
"""Start a new kernel.
The caller can pick a kernel_id by passing one in as a keyword arg,
otherwise one will be generated using new_kernel_id().
The kernel ID for the newly started kernel is returned.
"""
km, kernel_name, kernel_id = self.pre_start_kernel(kernel_name, kwargs)
km.start_kernel(**kwargs)
self._kernels[kernel_id] = km
return kernel_id
def shutdown_kernel(self, kernel_id, now=False, restart=False):
"""Shutdown a kernel by its kernel uuid.
Parameters
==========
kernel_id : uuid
The id of the kernel to shutdown.
now : bool
Should the kernel be shutdown forcibly using a signal.
restart : bool
Will the kernel be restarted?
"""
self.log.info("Kernel shutdown: %s" % kernel_id)
km = self.get_kernel(kernel_id)
ports = (km.shell_port, km.iopub_port, km.stdin_port, km.hb_port,
km.control_port)
km.shutdown_kernel(now=now, restart=restart)
self.remove_kernel(kernel_id)
if km.cache_ports and not restart:
for port in ports:
self.currently_used_ports.remove(port)
@kernel_method
def request_shutdown(self, kernel_id, restart=False):
"""Ask a kernel to shut down by its kernel uuid"""
@kernel_method
def finish_shutdown(self, kernel_id, waittime=None, pollinterval=0.1):
"""Wait for a kernel to finish shutting down, and kill it if it doesn't
"""
self.log.info("Kernel shutdown: %s" % kernel_id)
@kernel_method
def cleanup(self, kernel_id, connection_file=True):
"""Clean up a kernel's resources"""
@kernel_method
def cleanup_resources(self, kernel_id, restart=False):
"""Clean up a kernel's resources"""
def remove_kernel(self, kernel_id):
"""remove a kernel from our mapping.
Mainly so that a kernel can be removed if it is already dead,
without having to call shutdown_kernel.
The kernel object is returned.
"""
return self._kernels.pop(kernel_id)
def shutdown_all(self, now=False):
"""Shutdown all kernels."""
kids = self.list_kernel_ids()
for kid in kids:
self.request_shutdown(kid)
for kid in kids:
self.finish_shutdown(kid)
self.cleanup(kid)
self.remove_kernel(kid)
@kernel_method
def interrupt_kernel(self, kernel_id):
"""Interrupt (SIGINT) the kernel by its uuid.
Parameters
==========
kernel_id : uuid
The id of the kernel to interrupt.
"""
self.log.info("Kernel interrupted: %s" % kernel_id)
@kernel_method
def signal_kernel(self, kernel_id, signum):
"""Sends a signal to the kernel by its uuid.
Note that since only SIGTERM is supported on Windows, this function
is only useful on Unix systems.
Parameters
==========
kernel_id : uuid
The id of the kernel to signal.
"""
self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum))
@kernel_method
def restart_kernel(self, kernel_id, now=False):
"""Restart a kernel by its uuid, keeping the same ports.
Parameters
==========
kernel_id : uuid
The id of the kernel to interrupt.
"""
self.log.info("Kernel restarted: %s" % kernel_id)
@kernel_method
def is_alive(self, kernel_id):
"""Is the kernel alive.
This calls KernelManager.is_alive() which calls Popen.poll on the
actual kernel subprocess.
Parameters
==========
kernel_id : uuid
The id of the kernel.
"""
def _check_kernel_id(self, kernel_id):
"""check that a kernel id is valid"""
if kernel_id not in self:
raise KeyError("Kernel with id not found: %s" % kernel_id)
def get_kernel(self, kernel_id):
"""Get the single KernelManager object for a kernel by its uuid.
Parameters
==========
kernel_id : uuid
The id of the kernel.
"""
self._check_kernel_id(kernel_id)
return self._kernels[kernel_id]
@kernel_method
def add_restart_callback(self, kernel_id, callback, event='restart'):
"""add a callback for the KernelRestarter"""
@kernel_method
def remove_restart_callback(self, kernel_id, callback, event='restart'):
"""remove a callback for the KernelRestarter"""
@kernel_method
def get_connection_info(self, kernel_id):
"""Return a dictionary of connection data for a kernel.
Parameters
==========
kernel_id : uuid
The id of the kernel.
Returns
=======
connection_dict : dict
A dict of the information needed to connect to a kernel.
This includes the ip address and the integer port
numbers of the different channels (stdin_port, iopub_port,
shell_port, hb_port).
"""
@kernel_method
def connect_iopub(self, kernel_id, identity=None):
"""Return a zmq Socket connected to the iopub channel.
Parameters
==========
kernel_id : uuid
The id of the kernel
identity : bytes (optional)
The zmq identity of the socket
Returns
=======
stream : zmq Socket or ZMQStream
"""
@kernel_method
def connect_shell(self, kernel_id, identity=None):
"""Return a zmq Socket connected to the shell channel.
Parameters
==========
kernel_id : uuid
The id of the kernel
identity : bytes (optional)
The zmq identity of the socket
Returns
=======
stream : zmq Socket or ZMQStream
"""
@kernel_method
def connect_control(self, kernel_id, identity=None):
"""Return a zmq Socket connected to the control channel.
Parameters
==========
kernel_id : uuid
The id of the kernel
identity : bytes (optional)
The zmq identity of the socket
Returns
=======
stream : zmq Socket or ZMQStream
"""
@kernel_method
def connect_stdin(self, kernel_id, identity=None):
"""Return a zmq Socket connected to the stdin channel.
Parameters
==========
kernel_id : uuid
The id of the kernel
identity : bytes (optional)
The zmq identity of the socket
Returns
=======
stream : zmq Socket or ZMQStream
"""
@kernel_method
def connect_hb(self, kernel_id, identity=None):
"""Return a zmq Socket connected to the hb channel.
Parameters
==========
kernel_id : uuid
The id of the kernel
identity : bytes (optional)
The zmq identity of the socket
Returns
=======
stream : zmq Socket or ZMQStream
"""
def new_kernel_id(self, **kwargs):
"""
Returns the id to associate with the kernel for this request. Subclasses may override
this method to substitute other sources of kernel ids.
:param kwargs:
:return: string-ized version 4 uuid
"""
return unicode_type(uuid.uuid4())
class ZMQInteractiveShell(InteractiveShell):
"""A subclass of InteractiveShell for ZMQ."""
displayhook_class = Type(ZMQShellDisplayHook)
display_pub_class = Type(ZMQDisplayPublisher)
data_pub_class = Type('ibpykernel.datapub.ZMQDataPublisher')
kernel = Any()
parent_header = Any()
@default('banner1')
def _default_banner1(self):
return brackets.shell.BANNER
# Override the traitlet in the parent class, because there's no point using
# readline for the kernel. Can be removed when the readline code is moved
# to the terminal frontend.
colors_force = CBool(True)
readline_use = CBool(False)
# autoindent has no meaning in a zmqshell, and attempting to enable it
# will print a warning in the absence of readline.
autoindent = CBool(False)
exiter = Instance(ZMQExitAutocall)
@default('exiter')
def _default_exiter(self):
return ZMQExitAutocall(self)
@observe('exit_now')
def _update_exit_now(self, change):
"""stop eventloop when exit_now fires"""
if change['new']:
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time() + 0.1, loop.stop)
keepkernel_on_exit = None
# Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
# interactive input being read; we provide event loop support in ipkernel
def enable_gui(self, gui):
from .eventloops import enable_gui as real_enable_gui
try:
real_enable_gui(gui)
self.active_eventloop = gui
except ValueError as e:
raise UsageError("%s" % e)
def init_environment(self):
"""Configure the user's environment."""
env = os.environ
# These two ensure 'ls' produces nice coloring on BSD-derived systems
env['TERM'] = 'xterm-color'
env['CLICOLOR'] = '1'
# Since normal pagers don't work at all (over pexpect we don't have
# single-key control of the subprocess), try to disable paging in
# subprocesses as much as possible.
env['PAGER'] = 'cat'
env['GIT_PAGER'] = 'cat'
def init_hooks(self):
super(ZMQInteractiveShell, self).init_hooks()
self.set_hook('show_in_pager', page.as_hook(payloadpage.page), 99)
def init_data_pub(self):
"""Delay datapub init until request, for deprecation warnings"""
pass
@property
def data_pub(self):
if not hasattr(self, '_data_pub'):
warnings.warn(
"InteractiveShell.data_pub is deprecated outside IPython parallel.",
DeprecationWarning,
stacklevel=2)
self._data_pub = self.data_pub_class(parent=self)
self._data_pub.session = self.display_pub.session
self._data_pub.pub_socket = self.display_pub.pub_socket
return self._data_pub
@data_pub.setter
def data_pub(self, pub):
self._data_pub = pub
def ask_exit(self):
"""Engage the exit actions."""
self.exit_now = (not self.keepkernel_on_exit)
payload = dict(
source='ask_exit',
keepkernel=self.keepkernel_on_exit,
)
self.payload_manager.write_payload(payload)
def run_cell(self, *args, **kwargs):
self._last_traceback = None
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
def _showtraceback(self, etype, evalue, stb):
# try to preserve ordering of tracebacks and print statements
sys.stdout.flush()
sys.stderr.flush()
exc_content = {
u'traceback': stb,
u'ename': unicode_type(etype.__name__),
u'evalue': py3compat.safe_unicode(evalue),
}
dh = self.displayhook
# Send exception info over pub socket for other clients than the caller
# to pick up
topic = None
if dh.topic:
topic = dh.topic.replace(b'execute_result', b'error')
exc_msg = dh.session.send(dh.pub_socket,
u'error',
json_clean(exc_content),
dh.parent_header,
ident=topic)
# FIXME - Once we rely on Python 3, the traceback is stored on the
# exception object, so we shouldn't need to store it here.
self._last_traceback = stb
def set_next_input(self, text, replace=False):
"""Send the specified text to the frontend to be presented at the next
input cell."""
payload = dict(
source='set_next_input',
text=text,
replace=replace,
)
self.payload_manager.write_payload(payload)
def set_parent(self, parent):
"""Set the parent header for associating output with its triggering input"""
self.parent_header = parent
self.displayhook.set_parent(parent)
self.display_pub.set_parent(parent)
if hasattr(self, '_data_pub'):
self.data_pub.set_parent(parent)
try:
sys.stdout.set_parent(parent)
except AttributeError:
pass
try:
sys.stderr.set_parent(parent)
except AttributeError:
pass
def get_parent(self):
return self.parent_header
def init_magics(self):
super(ZMQInteractiveShell, self).init_magics()
self.register_magics(KernelMagics)
self.magics_manager.register_alias('ed', 'edit')
def init_virtualenv(self):
# Overridden not to do virtualenv detection, because it's probably
# not appropriate in a kernel. To use a kernel in a virtualenv, install
# it inside the virtualenv.
# https://ipython.readthedocs.io/en/latest/install/kernel_install.html
pass
class MappingKernelManager(MultiKernelManager):
"""A KernelManager that handles
- File mapping
- HTTP error handling
- Kernel message filtering
"""
@default('kernel_manager_class')
def _default_kernel_manager_class(self):
return "jupyter_client.ioloop.IOLoopKernelManager"
kernel_argv = List(Unicode())
root_dir = Unicode(config=True)
_kernel_connections = Dict()
_kernel_ports = Dict()
_culler_callback = None
_initialized_culler = False
@default('root_dir')
def _default_root_dir(self):
try:
return self.parent.root_dir
except AttributeError:
return os.getcwd()
@validate('root_dir')
def _update_root_dir(self, proposal):
"""Do a bit of validation of the root dir."""
value = proposal['value']
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not exists(value) or not os.path.isdir(value):
raise TraitError("kernel root dir %r is not a directory" % value)
return value
cull_idle_timeout = Integer(
0,
config=True,
help=
"""Timeout (in seconds) after which a kernel is considered idle and ready to be culled.
Values of 0 or lower disable culling. Very short timeouts may result in kernels being culled
for users with poor network connections.""")
cull_interval_default = 300 # 5 minutes
cull_interval = Integer(
cull_interval_default,
config=True,
help=
"""The interval (in seconds) on which to check for idle kernels exceeding the cull timeout value."""
)
cull_connected = Bool(
False,
config=True,
help=
"""Whether to consider culling kernels which have one or more connections.
Only effective if cull_idle_timeout > 0.""")
cull_busy = Bool(
False,
config=True,
help="""Whether to consider culling kernels which are busy.
Only effective if cull_idle_timeout > 0.""")
buffer_offline_messages = Bool(
True,
config=True,
help=
"""Whether messages from kernels whose frontends have disconnected should be buffered in-memory.
When True (default), messages are buffered and replayed on reconnect,
avoiding lost messages due to interrupted connectivity.
Disable if long-running kernels will produce too much output while
no frontends are connected.
""")
kernel_info_timeout = Float(
60,
config=True,
help="""Timeout for giving up on a kernel (in seconds).
On starting and restarting kernels, we check whether the
kernel is running and responsive by sending kernel_info_requests.
This sets the timeout in seconds for how long the kernel can take
before being presumed dead.
This affects the MappingKernelManager (which handles kernel restarts)
and the ZMQChannelsHandler (which handles the startup).
""")
_kernel_buffers = Any()
@default('_kernel_buffers')
def _default_kernel_buffers(self):
return defaultdict(lambda: {
'buffer': [],
'session_key': '',
'channels': {}
})
last_kernel_activity = Instance(
datetime,
help="The last activity on any kernel, including shutting down a kernel"
)
def __init__(self, **kwargs):
self.pinned_superclass = MultiKernelManager
self.pinned_superclass.__init__(self, **kwargs)
self.last_kernel_activity = utcnow()
allowed_message_types = List(
trait=Unicode(),
config=True,
help="""White list of allowed kernel message types.
When the list is empty, all message types are allowed.
""")
allow_tracebacks = Bool(
True,
config=True,
help=('Whether to send tracebacks to clients on exceptions.'))
traceback_replacement_message = Unicode(
'An exception occurred at runtime, which is not shown due to security reasons.',
config=True,
help=
('Message to print when allow_tracebacks is False, and an exception occurs'
))
#-------------------------------------------------------------------------
# Methods for managing kernels and sessions
#-------------------------------------------------------------------------
def _handle_kernel_died(self, kernel_id):
"""notice that a kernel died"""
self.log.warning("Kernel %s died, removing from map.", kernel_id)
self.remove_kernel(kernel_id)
def cwd_for_path(self, path):
"""Turn API path into absolute OS path."""
os_path = to_os_path(path, self.root_dir)
# in the case of documents and kernels not being on the same filesystem,
# walk up to root_dir if the paths don't exist
while not os.path.isdir(os_path) and os_path != self.root_dir:
os_path = os.path.dirname(os_path)
return os_path
async def start_kernel(self, kernel_id=None, path=None, **kwargs):
"""Start a kernel for a session and return its kernel_id.
Parameters
----------
kernel_id : uuid
The uuid to associate the new kernel with. If this
is not None, this kernel will be persistent whenever it is
requested.
path : API path
The API path (unicode, '/' delimited) for the cwd.
Will be transformed to an OS path relative to root_dir.
kernel_name : str
The name identifying which kernel spec to launch. This is ignored if
an existing kernel is returned, but it may be checked in the future.
"""
if kernel_id is None or kernel_id not in self:
if path is not None:
kwargs['cwd'] = self.cwd_for_path(path)
if kernel_id is not None:
kwargs['kernel_id'] = kernel_id
kernel_id = await ensure_async(
self.pinned_superclass.start_kernel(self, **kwargs))
self._kernel_connections[kernel_id] = 0
self._kernel_ports[kernel_id] = self._kernels[kernel_id].ports
self.start_watching_activity(kernel_id)
self.log.info("Kernel started: %s" % kernel_id)
self.log.debug("Kernel args: %r" % kwargs)
# register callback for failed auto-restart
self.add_restart_callback(
kernel_id,
lambda: self._handle_kernel_died(kernel_id),
'dead',
)
# Increase the metric of number of kernels running
# for the relevant kernel type by 1
KERNEL_CURRENTLY_RUNNING_TOTAL.labels(
type=self._kernels[kernel_id].kernel_name).inc()
else:
self.log.info("Using existing kernel: %s" % kernel_id)
# Initialize culling if not already
if not self._initialized_culler:
self.initialize_culler()
return kernel_id
def ports_changed(self, kernel_id):
"""Used by ZMQChannelsHandler to determine how to coordinate nudge and replays.
Ports are captured when starting a kernel (via MappingKernelManager). Ports
are considered changed (following restarts) if the referenced KernelManager
is using a set of ports different from those captured at startup. If changes
are detected, the captured set is updated and a value of True is returned.
NOTE: Use is exclusive to ZMQChannelsHandler because this object is a singleton
instance while ZMQChannelsHandler instances are per WebSocket connection that
can vary per kernel lifetime.
"""
changed_ports = self._get_changed_ports(kernel_id)
if changed_ports:
# If changed, update captured ports and return True, else return False.
self.log.debug(f"Port change detected for kernel: {kernel_id}")
self._kernel_ports[kernel_id] = changed_ports
return True
return False
def _get_changed_ports(self, kernel_id):
"""Internal method to test if a kernel's ports have changed and, if so, return their values.
This method does NOT update the captured ports for the kernel as that can only be done
by ZMQChannelsHandler, but instead returns the new list of ports if they are different
than those captured at startup. This enables the ability to conditionally restart
activity monitoring immediately following a kernel's restart (if ports have changed).
"""
# Get current ports and return comparison with ports captured at startup.
km = self.get_kernel(kernel_id)
if km.ports != self._kernel_ports[kernel_id]:
return km.ports
return None
def start_buffering(self, kernel_id, session_key, channels):
"""Start buffering messages for a kernel
Parameters
----------
kernel_id : str
The id of the kernel to stop buffering.
session_key : str
The session_key, if any, that should get the buffer.
If the session_key matches the current buffered session_key,
the buffer will be returned.
channels : dict({'channel': ZMQStream})
The zmq channels whose messages should be buffered.
"""
if not self.buffer_offline_messages:
for channel, stream in channels.items():
stream.close()
return
self.log.info("Starting buffering for %s", session_key)
self._check_kernel_id(kernel_id)
# clear previous buffering state
self.stop_buffering(kernel_id)
buffer_info = self._kernel_buffers[kernel_id]
# record the session key because only one session can buffer
buffer_info['session_key'] = session_key
# TODO: the buffer should likely be a memory bounded queue, we're starting with a list to keep it simple
buffer_info['buffer'] = []
buffer_info['channels'] = channels
# forward any future messages to the internal buffer
def buffer_msg(channel, msg_parts):
self.log.debug("Buffering msg on %s:%s", kernel_id, channel)
buffer_info['buffer'].append((channel, msg_parts))
for channel, stream in channels.items():
stream.on_recv(partial(buffer_msg, channel))
def get_buffer(self, kernel_id, session_key):
"""Get the buffer for a given kernel
Parameters
----------
kernel_id : str
The id of the kernel to stop buffering.
session_key : str, optional
The session_key, if any, that should get the buffer.
If the session_key matches the current buffered session_key,
the buffer will be returned.
"""
self.log.debug("Getting buffer for %s", kernel_id)
if kernel_id not in self._kernel_buffers:
return
buffer_info = self._kernel_buffers[kernel_id]
if buffer_info['session_key'] == session_key:
# remove buffer
self._kernel_buffers.pop(kernel_id)
# only return buffer_info if it's a match
return buffer_info
else:
self.stop_buffering(kernel_id)
def stop_buffering(self, kernel_id):
"""Stop buffering kernel messages
Parameters
----------
kernel_id : str
The id of the kernel to stop buffering.
"""
self.log.debug("Clearing buffer for %s", kernel_id)
self._check_kernel_id(kernel_id)
if kernel_id not in self._kernel_buffers:
return
buffer_info = self._kernel_buffers.pop(kernel_id)
# close buffering streams
for stream in buffer_info['channels'].values():
if not stream.closed():
stream.on_recv(None)
stream.close()
msg_buffer = buffer_info['buffer']
if msg_buffer:
self.log.info("Discarding %s buffered messages for %s",
len(msg_buffer), buffer_info['session_key'])
def shutdown_kernel(self, kernel_id, now=False, restart=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
self.stop_watching_activity(kernel_id)
self.stop_buffering(kernel_id)
self._kernel_connections.pop(kernel_id, None)
# Decrease the metric of number of kernels
# running for the relevant kernel type by 1
KERNEL_CURRENTLY_RUNNING_TOTAL.labels(
type=self._kernels[kernel_id].kernel_name).dec()
self.pinned_superclass.shutdown_kernel(self,
kernel_id,
now=now,
restart=restart)
# Unlike its async sibling method in AsyncMappingKernelManager, removing the kernel_id
# from the connections dictionary isn't as problematic before the shutdown since the
# method is synchronous. However, we'll keep the relative call orders the same from
# a maintenance perspective.
self._kernel_connections.pop(kernel_id, None)
self._kernel_ports.pop(kernel_id, None)
async def restart_kernel(self, kernel_id, now=False):
"""Restart a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
await ensure_async(
self.pinned_superclass.restart_kernel(self, kernel_id, now=now))
kernel = self.get_kernel(kernel_id)
# return a Future that will resolve when the kernel has successfully restarted
channel = kernel.connect_shell()
future = Future()
def finish():
"""Common cleanup when restart finishes/fails for any reason."""
if not channel.closed():
channel.close()
loop.remove_timeout(timeout)
kernel.remove_restart_callback(on_restart_failed, 'dead')
def on_reply(msg):
self.log.debug("Kernel info reply received: %s", kernel_id)
finish()
if not future.done():
future.set_result(msg)
def on_timeout():
self.log.warning("Timeout waiting for kernel_info_reply: %s",
kernel_id)
finish()
if not future.done():
future.set_exception(
TimeoutError("Timeout waiting for restart"))
def on_restart_failed():
self.log.warning("Restarting kernel failed: %s", kernel_id)
finish()
if not future.done():
future.set_exception(RuntimeError("Restart failed"))
kernel.add_restart_callback(on_restart_failed, 'dead')
kernel.session.send(channel, "kernel_info_request")
channel.on_recv(on_reply)
loop = IOLoop.current()
timeout = loop.add_timeout(loop.time() + self.kernel_info_timeout,
on_timeout)
# Re-establish activity watching if ports have changed...
if self._get_changed_ports(kernel_id) is not None:
self.stop_watching_activity(kernel_id)
self.start_watching_activity(kernel_id)
return future
def notify_connect(self, kernel_id):
"""Notice a new connection to a kernel"""
if kernel_id in self._kernel_connections:
self._kernel_connections[kernel_id] += 1
def notify_disconnect(self, kernel_id):
"""Notice a disconnection from a kernel"""
if kernel_id in self._kernel_connections:
self._kernel_connections[kernel_id] -= 1
def kernel_model(self, kernel_id):
"""Return a JSON-safe dict representing a kernel
For use in representing kernels in the JSON APIs.
"""
self._check_kernel_id(kernel_id)
kernel = self._kernels[kernel_id]
model = {
"id": kernel_id,
"name": kernel.kernel_name,
"last_activity": isoformat(kernel.last_activity),
"execution_state": kernel.execution_state,
"connections": self._kernel_connections.get(kernel_id, 0),
}
return model
def list_kernels(self):
"""Returns a list of kernel_id's of kernels running."""
kernels = []
kernel_ids = self.pinned_superclass.list_kernel_ids(self)
for kernel_id in kernel_ids:
try:
model = self.kernel_model(kernel_id)
kernels.append(model)
except (web.HTTPError, KeyError):
pass # Probably due to a (now) non-existent kernel, continue building the list
return kernels
# override _check_kernel_id to raise 404 instead of KeyError
def _check_kernel_id(self, kernel_id):
"""Check a that a kernel_id exists and raise 404 if not."""
if kernel_id not in self:
raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
# monitoring activity:
def start_watching_activity(self, kernel_id):
"""Start watching IOPub messages on a kernel for activity.
- update last_activity on every message
- record execution_state from status messages
"""
kernel = self._kernels[kernel_id]
# add busy/activity markers:
kernel.execution_state = 'starting'
kernel.last_activity = utcnow()
kernel._activity_stream = kernel.connect_iopub()
session = Session(
config=kernel.session.config,
key=kernel.session.key,
)
def record_activity(msg_list):
"""Record an IOPub message arriving from a kernel"""
self.last_kernel_activity = kernel.last_activity = utcnow()
idents, fed_msg_list = session.feed_identities(msg_list)
msg = session.deserialize(fed_msg_list)
msg_type = msg['header']['msg_type']
if msg_type == 'status':
kernel.execution_state = msg['content']['execution_state']
self.log.debug("activity on %s: %s (%s)", kernel_id, msg_type,
kernel.execution_state)
else:
self.log.debug("activity on %s: %s", kernel_id, msg_type)
kernel._activity_stream.on_recv(record_activity)
def stop_watching_activity(self, kernel_id):
"""Stop watching IOPub messages on a kernel for activity."""
kernel = self._kernels[kernel_id]
if kernel._activity_stream:
kernel._activity_stream.close()
kernel._activity_stream = None
def initialize_culler(self):
"""Start idle culler if 'cull_idle_timeout' is greater than zero.
Regardless of that value, set flag that we've been here.
"""
if not self._initialized_culler and self.cull_idle_timeout > 0:
if self._culler_callback is None:
loop = IOLoop.current()
if self.cull_interval <= 0: #handle case where user set invalid value
self.log.warning(
"Invalid value for 'cull_interval' detected (%s) - using default value (%s).",
self.cull_interval, self.cull_interval_default)
self.cull_interval = self.cull_interval_default
self._culler_callback = PeriodicCallback(
self.cull_kernels, 1000 * self.cull_interval)
self.log.info(
"Culling kernels with idle durations > %s seconds at %s second intervals ...",
self.cull_idle_timeout, self.cull_interval)
if self.cull_busy:
self.log.info("Culling kernels even if busy")
if self.cull_connected:
self.log.info(
"Culling kernels even with connected clients")
self._culler_callback.start()
self._initialized_culler = True
async def cull_kernels(self):
self.log.debug(
"Polling every %s seconds for kernels idle > %s seconds...",
self.cull_interval, self.cull_idle_timeout)
"""Create a separate list of kernels to avoid conflicting updates while iterating"""
for kernel_id in list(self._kernels):
try:
await self.cull_kernel_if_idle(kernel_id)
except Exception as e:
self.log.exception(
"The following exception was encountered while checking the idle duration of kernel %s: %s",
kernel_id, e)
async def cull_kernel_if_idle(self, kernel_id):
kernel = self._kernels[kernel_id]
if hasattr(
kernel, 'last_activity'
): # last_activity is monkey-patched, so ensure that has occurred
self.log.debug("kernel_id=%s, kernel_name=%s, last_activity=%s",
kernel_id, kernel.kernel_name, kernel.last_activity)
dt_now = utcnow()
dt_idle = dt_now - kernel.last_activity
# Compute idle properties
is_idle_time = dt_idle > timedelta(seconds=self.cull_idle_timeout)
is_idle_execute = self.cull_busy or (kernel.execution_state !=
'busy')
connections = self._kernel_connections.get(kernel_id, 0)
is_idle_connected = self.cull_connected or not connections
# Cull the kernel if all three criteria are met
if (is_idle_time and is_idle_execute and is_idle_connected):
idle_duration = int(dt_idle.total_seconds())
self.log.warning(
"Culling '%s' kernel '%s' (%s) with %d connections due to %s seconds of inactivity.",
kernel.execution_state, kernel.kernel_name, kernel_id,
connections, idle_duration)
await ensure_async(self.shutdown_kernel(kernel_id))
class InProcessKernel(IPythonKernel):
#-------------------------------------------------------------------------
# InProcessKernel interface
#-------------------------------------------------------------------------
# The frontends connected to this kernel.
frontends = List(
Instance('ipykernel.inprocess.client.InProcessKernelClient',
allow_none=True))
# The GUI environment that the kernel is running under. This need not be
# specified for the normal operation for the kernel, but is required for
# IPython's GUI support (including pylab). The default is 'inline' because
# it is safe under all GUI toolkits.
gui = Enum(('tk', 'gtk', 'wx', 'qt', 'qt4', 'inline'),
default_value='inline')
raw_input_str = Any()
stdout = Any()
stderr = Any()
#-------------------------------------------------------------------------
# Kernel interface
#-------------------------------------------------------------------------
shell_class = Type(allow_none=True)
shell_streams = List()
control_stream = Any()
iopub_socket = Instance(DummySocket, ())
stdin_socket = Instance(DummySocket, ())
def __init__(self, **traits):
super(InProcessKernel, self).__init__(**traits)
self.iopub_socket.on_trait_change(self._io_dispatch, 'message_sent')
self.shell.kernel = self
def execute_request(self, stream, ident, parent):
""" Override for temporary IO redirection. """
with self._redirected_io():
super(InProcessKernel, self).execute_request(stream, ident, parent)
def start(self):
""" Override registration of dispatchers for streams. """
self.shell.exit_now = False
def _abort_queue(self, stream):
""" The in-process kernel doesn't abort requests. """
pass
def _input_request(self, prompt, ident, parent, password=False):
# Flush output before making the request.
self.raw_input_str = None
sys.stderr.flush()
sys.stdout.flush()
# Send the input request.
content = json_clean(dict(prompt=prompt, password=password))
msg = self.session.msg(u'input_request', content, parent)
for frontend in self.frontends:
if frontend.session.session == parent['header']['session']:
frontend.stdin_channel.call_handlers(msg)
break
else:
logging.error('No frontend found for raw_input request')
return str()
# Await a response.
while self.raw_input_str is None:
frontend.stdin_channel.process_events()
return self.raw_input_str
#-------------------------------------------------------------------------
# Protected interface
#-------------------------------------------------------------------------
@contextmanager
def _redirected_io(self):
""" Temporarily redirect IO to the kernel.
"""
sys_stdout, sys_stderr = sys.stdout, sys.stderr
sys.stdout, sys.stderr = self.stdout, self.stderr
yield
sys.stdout, sys.stderr = sys_stdout, sys_stderr
#------ Trait change handlers --------------------------------------------
def _io_dispatch(self):
""" Called when a message is sent to the IO socket.
"""
ident, msg = self.session.recv(self.iopub_socket, copy=False)
for frontend in self.frontends:
frontend.iopub_channel.call_handlers(msg)
#------ Trait initializers -----------------------------------------------
def _log_default(self):
return logging.getLogger(__name__)
def _session_default(self):
from jupyter_client.session import Session
return Session(parent=self, key=b'')
def _shell_class_default(self):
return InProcessInteractiveShell
def _stdout_default(self):
from ipykernel.iostream import OutStream
return OutStream(self.session,
self.iopub_socket,
u'stdout',
pipe=False)
def _stderr_default(self):
from ipykernel.iostream import OutStream
return OutStream(self.session,
self.iopub_socket,
u'stderr',
pipe=False)
class MWOAuthenticator(OAuthenticator):
login_service = 'MediaWiki'
login_handler = MWLoginHandler
callback_handler = MWCallbackHandler
mw_index_url = Unicode(
os.environ.get('MW_INDEX_URL', 'https://meta.wikimedia.org/w/index.php'),
config=True,
help='Full path to index.php of the MW instance to use to log in'
)
executor_threads = Integer(12,
help="""Number of executor threads.
MediaWiki OAuth requests happen in this thread,
so it is mostly waiting for network replies.
""",
config=True,
)
executor = Any()
def normalize_username(self, username):
"""
Override normalize_username to avoid lowercasing usernames
"""
return username
def _executor_default(self):
return ThreadPoolExecutor(self.executor_threads)
@gen.coroutine
def authenticate(self, handler, data=None):
consumer_token = ConsumerToken(
self.client_id,
self.client_secret,
)
handshaker = Handshaker(
self.mw_index_url, consumer_token
)
request_token = dejsonify(handler.get_secure_cookie(AUTH_REQUEST_COOKIE_NAME))
handler.clear_cookie(AUTH_REQUEST_COOKIE_NAME)
access_token = yield self.executor.submit(
handshaker.complete, request_token, handler.request.query
)
identity = yield self.executor.submit(handshaker.identify, access_token)
if identity and 'username' in identity:
# this shouldn't be necessary anymore,
# but keep for backward-compatibility
return {
'name': identity['username'].replace(' ', '_'),
'auth_state': {
'ACCESS_TOKEN_KEY': access_token.key.decode('utf-8'),
'ACCESS_TOKEN_SECRET': access_token.secret.decode('utf-8'),
'MEDIAWIKI_USER_IDENTITY': identity,
}
}
else:
self.log.error("No username found in %s", identity)
class ZMQTerminalInteractiveShell(SingletonConfigurable):
readline_use = False
pt_cli = None
_executing = False
_execution_state = Unicode('')
_pending_clearoutput = False
_eventloop = None
editing_mode = Unicode(
'emacs',
config=True,
help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
)
highlighting_style = Unicode(
'',
config=True,
help="The name of a Pygments style to use for syntax highlighting")
highlighting_style_overrides = Dict(
config=True, help="Override highlighting format for specific tokens")
true_color = Bool(
False,
config=True,
help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
"If your terminal supports true color, the following command "
"should print 'TRUECOLOR' in orange: "
"printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\""))
history_load_length = Integer(
1000, config=True, help="How many history items to load into memory")
banner = Unicode(
'Jupyter console {version}\n\n{kernel_banner}',
config=True,
help=(
"Text to display before the first prompt. Will be formatted with "
"variables {version} and {kernel_banner}."))
kernel_timeout = Float(
60,
config=True,
help="""Timeout for giving up on a kernel (in seconds).
On first connect and restart, the console tests whether the
kernel is running and responsive by sending kernel_info_requests.
This sets the timeout in seconds for how long the kernel can take
before being presumed dead.
""")
image_handler = Enum(('PIL', 'stream', 'tempfile', 'callable'),
'PIL',
config=True,
allow_none=True,
help="""
Handler for image type output. This is useful, for example,
when connecting to the kernel in which pylab inline backend is
activated. There are four handlers defined. 'PIL': Use
Python Imaging Library to popup image; 'stream': Use an
external program to show the image. Image will be fed into
the STDIN of the program. You will need to configure
`stream_image_handler`; 'tempfile': Use an external program to
show the image. Image will be saved in a temporally file and
the program is called with the temporally file. You will need
to configure `tempfile_image_handler`; 'callable': You can set
any Python callable which is called with the image data. You
will need to configure `callable_image_handler`.
""")
stream_image_handler = List(config=True,
help="""
Command to invoke an image viewer program when you are using
'stream' image handler. This option is a list of string where
the first element is the command itself and reminders are the
options for the command. Raw image data is given as STDIN to
the program.
""")
tempfile_image_handler = List(config=True,
help="""
Command to invoke an image viewer program when you are using
'tempfile' image handler. This option is a list of string
where the first element is the command itself and reminders
are the options for the command. You can use {file} and
{format} in the string to represent the location of the
generated image file and image format.
""")
callable_image_handler = Any(config=True,
help="""
Callable object called via 'callable' image handler with one
argument, `data`, which is `msg["content"]["data"]` where
`msg` is the message from iopub channel. For exmaple, you can
find base64 encoded PNG data as `data['image/png']`. If your function
can't handle the data supplied, it should return `False` to indicate
this.
""")
mime_preference = List(
default_value=['image/png', 'image/jpeg', 'image/svg+xml'],
config=True,
help="""
Preferred object representation MIME type in order. First
matched MIME type will be used.
""")
use_kernel_is_complete = Bool(
True,
config=True,
help="""Whether to use the kernel's is_complete message
handling. If False, then the frontend will use its
own is_complete handler.
""")
kernel_is_complete_timeout = Float(
1,
config=True,
help="""Timeout (in seconds) for giving up on a kernel's is_complete
response.
If the kernel does not respond at any point within this time,
the kernel will no longer be asked if code is complete, and the
console will default to the built-in is_complete test.
""")
confirm_exit = Bool(True,
config=True,
help="""Set to display confirmation dialog on exit.
You can always use 'exit' or 'quit', to force a
direct exit without any confirmation.
""")
manager = Instance('jupyter_client.KernelManager', allow_none=True)
client = Instance('jupyter_client.KernelClient', allow_none=True)
def _client_changed(self, name, old, new):
self.session_id = new.session.session
session_id = Unicode()
def _banner1_default(self):
return "Jupyter Console {version}\n".format(version=__version__)
def __init__(self, **kwargs):
# This is where traits with a config_key argument are updated
# from the values on config.
super(ZMQTerminalInteractiveShell, self).__init__(**kwargs)
self.configurables = [self]
self.init_history()
self.init_completer()
self.init_io()
self.init_kernel_info()
self.init_prompt_toolkit_cli()
self.keep_running = True
self.execution_count = 1
def init_completer(self):
"""Initialize the completion machinery.
This creates completion machinery that can be used by client code,
either interactively in-process (typically triggered by the readline
library), programmatically (such as in test suites) or out-of-process
(typically over the network by remote frontends).
"""
self.Completer = ZMQCompleter(self, self.client, config=self.config)
def init_history(self):
"""Sets up the command history. """
self.history_manager = ZMQHistoryManager(client=self.client)
self.configurables.append(self.history_manager)
def get_prompt_tokens(self, cli):
return [
(Token.Prompt, 'In ['),
(Token.PromptNum, str(self.execution_count)),
(Token.Prompt, ']: '),
]
def get_continuation_tokens(self, cli, width):
return [
(Token.Prompt, (' ' * (width - 2)) + ': '),
]
def get_out_prompt_tokens(self):
return [(Token.OutPrompt, 'Out['),
(Token.OutPromptNum, str(self.execution_count)),
(Token.OutPrompt, ']: ')]
def print_out_prompt(self):
self.pt_cli.print_tokens(self.get_out_prompt_tokens())
kernel_info = {}
def init_kernel_info(self):
"""Wait for a kernel to be ready, and store kernel info"""
timeout = self.kernel_timeout
tic = time.time()
self.client.hb_channel.unpause()
msg_id = self.client.kernel_info()
while True:
try:
reply = self.client.get_shell_msg(timeout=1)
except Empty:
if (time.time() - tic) > timeout:
raise RuntimeError(
"Kernel didn't respond to kernel_info_request")
else:
if reply['parent_header'].get('msg_id') == msg_id:
self.kernel_info = reply['content']
return
def show_banner(self):
print(
self.banner.format(version=__version__,
kernel_banner=self.kernel_info.get(
'banner', '')))
def init_prompt_toolkit_cli(self):
if 'JUPYTER_CONSOLE_TEST' in os.environ:
# Simple restricted interface for tests so we can find prompts with
# pexpect. Multi-line input not supported.
def prompt():
return cast_unicode_py2(
input('In [%d]: ' % self.execution_count))
self.prompt_for_code = prompt
self.print_out_prompt = \
lambda: print('Out[%d]: ' % self.execution_count, end='')
return
kbmanager = KeyBindingManager.for_prompt()
insert_mode = ViInsertMode() | EmacsInsertMode()
# Ctrl+J == Enter, seemingly
@kbmanager.registry.add_binding(Keys.ControlJ,
filter=(HasFocus(DEFAULT_BUFFER)
& ~HasSelection()
& insert_mode))
def _(event):
b = event.current_buffer
d = b.document
if not (d.on_last_line or d.cursor_position_row >=
d.line_count - d.empty_line_count_at_the_end()):
b.newline()
return
more, indent = self.check_complete(d.text)
if (not more) and b.accept_action.is_returnable:
b.accept_action.validate_and_handle(event.cli, b)
else:
b.insert_text('\n' + indent)
@kbmanager.registry.add_binding(Keys.ControlC,
filter=HasFocus(DEFAULT_BUFFER))
def _(event):
event.current_buffer.reset()
# Pre-populate history from IPython's history database
history = InMemoryHistory()
last_cell = u""
for _, _, cell in self.history_manager.get_tail(
self.history_load_length, include_latest=True):
# Ignore blank lines and consecutive duplicates
cell = cell.rstrip()
if cell and (cell != last_cell):
history.append(cell)
style_overrides = {
Token.Prompt: '#009900',
Token.PromptNum: '#00ff00 bold',
Token.OutPrompt: '#ff2200',
Token.OutPromptNum: '#ff0000 bold',
}
if self.highlighting_style:
style_cls = get_style_by_name(self.highlighting_style)
else:
style_cls = get_style_by_name('default')
# The default theme needs to be visible on both a dark background
# and a light background, because we can't tell what the terminal
# looks like. These tweaks to the default theme help with that.
style_overrides.update({
Token.Number: '#007700',
Token.Operator: 'noinherit',
Token.String: '#BB6622',
Token.Name.Function: '#2080D0',
Token.Name.Class: 'bold #2080D0',
Token.Name.Namespace: 'bold #2080D0',
})
style_overrides.update(self.highlighting_style_overrides)
style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls,
style_dict=style_overrides)
editing_mode = getattr(EditingMode, self.editing_mode.upper())
langinfo = self.kernel_info.get('language_info', {})
lexer = langinfo.get('pygments_lexer', langinfo.get('name', 'text'))
app = create_prompt_application(
multiline=True,
editing_mode=editing_mode,
lexer=PygmentsLexer(get_pygments_lexer(lexer)),
get_prompt_tokens=self.get_prompt_tokens,
get_continuation_tokens=self.get_continuation_tokens,
key_bindings_registry=kbmanager.registry,
history=history,
completer=JupyterPTCompleter(self.Completer),
enable_history_search=True,
style=style,
)
self._eventloop = create_eventloop()
self.pt_cli = CommandLineInterface(
app,
eventloop=self._eventloop,
output=create_output(true_color=self.true_color),
)
def prompt_for_code(self):
document = self.pt_cli.run(pre_run=self.pre_prompt,
reset_current_buffer=True)
return document.text
def init_io(self):
if sys.platform not in {'win32', 'cli'}:
return
import colorama
colorama.init()
def check_complete(self, code):
if self.use_kernel_is_complete:
msg_id = self.client.is_complete(code)
try:
return self.handle_is_complete_reply(
msg_id, timeout=self.kernel_is_complete_timeout)
except SyntaxError:
return False, ""
else:
lines = code.splitlines()
if len(lines):
more = (lines[-1] != "")
return more, ""
else:
return False, ""
def ask_exit(self):
self.keep_running = False
# This is set from payloads in handle_execute_reply
next_input = None
def pre_prompt(self):
if self.next_input:
b = self.pt_cli.application.buffer
b.text = cast_unicode_py2(self.next_input)
self.next_input = None
# Move the cursor to the end
b.cursor_position += b.document.get_end_of_document_position()
def interact(self, display_banner=None):
while self.keep_running:
print('\n', end='')
try:
code = self.prompt_for_code()
except EOFError:
if (not self.confirm_exit) \
or ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
self.ask_exit()
else:
if code:
self.run_cell(code, store_history=True)
def mainloop(self):
self.keepkernel = False
# An extra layer of protection in case someone mashing Ctrl-C breaks
# out of our internal code.
while True:
try:
self.interact()
break
except KeyboardInterrupt:
print("\nKeyboardInterrupt escaped interact()\n")
self._eventloop.close()
if self.keepkernel and not self.own_kernel:
print('keeping kernel alive')
elif self.keepkernel and self.own_kernel:
print("owning kernel, cannot keep it alive")
self.client.shutdown()
else:
print("Shutting down kernel")
self.client.shutdown()
def run_cell(self, cell, store_history=True):
"""Run a complete IPython cell.
Parameters
----------
cell : str
The code (including IPython code such as %magic functions) to run.
store_history : bool
If True, the raw and translated cell will be stored in IPython's
history. For user code calling back into IPython's machinery, this
should be set to False.
"""
if (not cell) or cell.isspace():
# pressing enter flushes any pending display
self.handle_iopub()
return
# flush stale replies, which could have been ignored, due to missed heartbeats
while self.client.shell_channel.msg_ready():
self.client.shell_channel.get_msg()
# execute takes 'hidden', which is the inverse of store_hist
msg_id = self.client.execute(cell, not store_history)
# first thing is wait for any side effects (output, stdin, etc.)
self._executing = True
self._execution_state = "busy"
while self._execution_state != 'idle' and self.client.is_alive():
try:
self.handle_input_request(msg_id, timeout=0.05)
except Empty:
# display intermediate print statements, etc.
self.handle_iopub(msg_id)
except ZMQError as e:
# Carry on if polling was interrupted by a signal
if e.errno != errno.EINTR:
raise
# after all of that is done, wait for the execute reply
while self.client.is_alive():
try:
self.handle_execute_reply(msg_id, timeout=0.05)
except Empty:
pass
else:
break
self._executing = False
#-----------------
# message handlers
#-----------------
def handle_execute_reply(self, msg_id, timeout=None):
msg = self.client.shell_channel.get_msg(block=False, timeout=timeout)
if msg["parent_header"].get("msg_id", None) == msg_id:
self.handle_iopub(msg_id)
content = msg["content"]
status = content['status']
if status == 'aborted':
self.write('Aborted\n')
return
elif status == 'ok':
# handle payloads
for item in content.get("payload", []):
source = item['source']
if source == 'page':
page.page(item['data']['text/plain'])
elif source == 'set_next_input':
self.next_input = item['text']
elif source == 'ask_exit':
self.keepkernel = item.get('keepkernel', False)
self.ask_exit()
elif status == 'error':
pass
self.execution_count = int(content["execution_count"] + 1)
def handle_is_complete_reply(self, msg_id, timeout=None):
"""
Wait for a repsonse from the kernel, and return two values:
more? - (boolean) should the frontend ask for more input
indent - an indent string to prefix the input
Overloaded methods may want to examine the comeplete source. Its is
in the self._source_lines_buffered list.
"""
## Get the is_complete response:
msg = None
try:
msg = self.client.shell_channel.get_msg(block=True,
timeout=timeout)
except Empty:
warn('The kernel did not respond to an is_complete_request. '
'Setting `use_kernel_is_complete` to False.')
self.use_kernel_is_complete = False
return False, ""
## Handle response:
if msg["parent_header"].get("msg_id", None) != msg_id:
warn(
'The kernel did not respond properly to an is_complete_request: %s.'
% str(msg))
return False, ""
else:
status = msg["content"].get("status", None)
indent = msg["content"].get("indent", "")
## Return more? and indent string
if status == "complete":
return False, indent
elif status == "incomplete":
return True, indent
elif status == "invalid":
raise SyntaxError()
elif status == "unknown":
return False, indent
else:
warn('The kernel sent an invalid is_complete_reply status: "%s".' %
status)
return False, indent
include_other_output = Bool(False,
config=True,
help="""Whether to include output from clients
other than this one sharing the same kernel.
Outputs are not displayed until enter is pressed.
""")
other_output_prefix = Unicode(
"[remote] ",
config=True,
help="""Prefix to add to outputs coming from clients other than this one.
Only relevant if include_other_output is True.
""")
def from_here(self, msg):
"""Return whether a message is from this session"""
return msg['parent_header'].get("session",
self.session_id) == self.session_id
def include_output(self, msg):
"""Return whether we should include a given output message"""
from_here = self.from_here(msg)
if msg['msg_type'] == 'execute_input':
# only echo inputs not from here
return self.include_other_output and not from_here
if self.include_other_output:
return True
else:
return from_here
def handle_iopub(self, msg_id=''):
"""Process messages on the IOPub channel
This method consumes and processes messages on the IOPub channel,
such as stdout, stderr, execute_result and status.
It only displays output that is caused by this session.
"""
while self.client.iopub_channel.msg_ready():
sub_msg = self.client.iopub_channel.get_msg()
msg_type = sub_msg['header']['msg_type']
parent = sub_msg["parent_header"]
if self.include_output(sub_msg):
if msg_type == 'status':
self._execution_state = sub_msg["content"][
"execution_state"]
elif msg_type == 'stream':
if sub_msg["content"]["name"] == "stdout":
if self._pending_clearoutput:
print("\r", end="")
self._pending_clearoutput = False
print(sub_msg["content"]["text"], end="")
sys.stdout.flush()
elif sub_msg["content"]["name"] == "stderr":
if self._pending_clearoutput:
print("\r", file=sys.stderr, end="")
self._pending_clearoutput = False
print(sub_msg["content"]["text"],
file=sys.stderr,
end="")
sys.stderr.flush()
elif msg_type == 'execute_result':
if self._pending_clearoutput:
print("\r", end="")
self._pending_clearoutput = False
self.execution_count = int(
sub_msg["content"]["execution_count"])
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
format_dict = sub_msg["content"]["data"]
self.handle_rich_data(format_dict)
if 'text/plain' not in format_dict:
continue
self.print_out_prompt()
text_repr = format_dict['text/plain']
if '\n' in text_repr:
# For multi-line results, start a new line after prompt
print()
print(text_repr)
elif msg_type == 'display_data':
data = sub_msg["content"]["data"]
handled = self.handle_rich_data(data)
if not handled:
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
# if it was an image, we handled it by now
if 'text/plain' in data:
print(data['text/plain'])
elif msg_type == 'execute_input':
content = sub_msg['content']
if not self.from_here(sub_msg):
sys.stdout.write(self.other_output_prefix)
sys.stdout.write('In [{}]: '.format(
content['execution_count']))
sys.stdout.write(content['code'])
elif msg_type == 'clear_output':
if sub_msg["content"]["wait"]:
self._pending_clearoutput = True
else:
print("\r", end="")
elif msg_type == 'error':
for frame in sub_msg["content"]["traceback"]:
print(frame, file=sys.stderr)
_imagemime = {
'image/png': 'png',
'image/jpeg': 'jpeg',
'image/svg+xml': 'svg',
}
def handle_rich_data(self, data):
for mime in self.mime_preference:
if mime in data and mime in self._imagemime:
if self.handle_image(data, mime):
return True
return False
def handle_image(self, data, mime):
handler = getattr(self, 'handle_image_{0}'.format(self.image_handler),
None)
if handler:
return handler(data, mime)
def handle_image_PIL(self, data, mime):
if mime not in ('image/png', 'image/jpeg'):
return False
try:
from PIL import Image, ImageShow
except ImportError:
return False
raw = base64.decodestring(data[mime].encode('ascii'))
img = Image.open(BytesIO(raw))
return ImageShow.show(img)
def handle_image_stream(self, data, mime):
raw = base64.decodestring(data[mime].encode('ascii'))
imageformat = self._imagemime[mime]
fmt = dict(format=imageformat)
args = [s.format(**fmt) for s in self.stream_image_handler]
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=devnull,
stderr=devnull)
proc.communicate(raw)
return (proc.returncode == 0)
def handle_image_tempfile(self, data, mime):
raw = base64.decodestring(data[mime].encode('ascii'))
imageformat = self._imagemime[mime]
filename = 'tmp.{0}'.format(imageformat)
with NamedFileInTemporaryDirectory(filename) as f, \
open(os.devnull, 'w') as devnull:
f.write(raw)
f.flush()
fmt = dict(file=f.name, format=imageformat)
args = [s.format(**fmt) for s in self.tempfile_image_handler]
rc = subprocess.call(args, stdout=devnull, stderr=devnull)
return (rc == 0)
def handle_image_callable(self, data, mime):
res = self.callable_image_handler(data)
if res is not False:
# If handler func returns e.g. None, assume it has handled the data.
res = True
return res
def handle_input_request(self, msg_id, timeout=0.1):
""" Method to capture raw_input
"""
req = self.client.stdin_channel.get_msg(timeout=timeout)
# in case any iopub came while we were waiting:
self.handle_iopub(msg_id)
if msg_id == req["parent_header"].get("msg_id"):
# wrap SIGINT handler
real_handler = signal.getsignal(signal.SIGINT)
def double_int(sig, frame):
# call real handler (forwards sigint to kernel),
# then raise local interrupt, stopping local raw_input
real_handler(sig, frame)
raise KeyboardInterrupt
signal.signal(signal.SIGINT, double_int)
content = req['content']
read = getpass if content.get('password', False) else input
try:
raw_data = read(content["prompt"])
except EOFError:
# turn EOFError into EOF character
raw_data = '\x04'
except KeyboardInterrupt:
sys.stdout.write('\n')
return
finally:
# restore SIGINT handler
signal.signal(signal.SIGINT, real_handler)
# only send stdin reply if there *was not* another request
# or execution finished while we were reading.
if not (self.client.stdin_channel.msg_ready()
or self.client.shell_channel.msg_ready()):
self.client.input(raw_data)
class JupyterHub(Application):
"""An Application for starting a Multi-User Jupyter Notebook server."""
name = 'jupyterhub'
version = jupyterhub.__version__
description = """Start a multi-user Jupyter Notebook server
Spawns a configurable-http-proxy and multi-user Hub,
which authenticates users and spawns single-user Notebook servers
on behalf of users.
"""
examples = """
generate default config file:
jupyterhub --generate-config -f /etc/jupyterhub/jupyterhub.py
spawn the server on 10.0.1.2:443 with https:
jupyterhub --ip 10.0.1.2 --port 443 --ssl-key my_ssl.key --ssl-cert my_ssl.cert
"""
aliases = Dict(aliases)
flags = Dict(flags)
subcommands = {'token': (NewToken, "Generate an API token for a user")}
classes = List([
Spawner,
LocalProcessSpawner,
Authenticator,
PAMAuthenticator,
])
config_file = Unicode(
'jupyterhub_config.py',
config=True,
help="The config file to load",
)
generate_config = Bool(
False,
config=True,
help="Generate default config file",
)
answer_yes = Bool(
False,
config=True,
help="Answer yes to any questions (e.g. confirm overwrite)")
pid_file = Unicode('',
config=True,
help="""File to write PID
Useful for daemonizing jupyterhub.
""")
cookie_max_age_days = Float(
14,
config=True,
help="""Number of days for a login cookie to be valid.
Default is two weeks.
""")
last_activity_interval = Integer(
300,
config=True,
help=
"Interval (in seconds) at which to update last-activity timestamps.")
proxy_check_interval = Integer(
30,
config=True,
help="Interval (in seconds) at which to check if the proxy is running."
)
data_files_path = Unicode(
DATA_FILES_PATH,
config=True,
help=
"The location of jupyterhub data files (e.g. /usr/local/share/jupyter/hub)"
)
template_paths = List(
config=True,
help="Paths to search for jinja templates.",
)
def _template_paths_default(self):
return [os.path.join(self.data_files_path, 'templates')]
ssl_key = Unicode(
'',
config=True,
help="""Path to SSL key file for the public facing interface of the proxy
Use with ssl_cert
""")
ssl_cert = Unicode(
'',
config=True,
help=
"""Path to SSL certificate file for the public facing interface of the proxy
Use with ssl_key
""")
ip = Unicode('', config=True, help="The public facing ip of the proxy")
port = Integer(8000,
config=True,
help="The public facing port of the proxy")
base_url = URLPrefix('/',
config=True,
help="The base URL of the entire application")
jinja_environment_options = Dict(
config=True,
help="Supply extra arguments that will be passed to Jinja environment."
)
proxy_cmd = Command('configurable-http-proxy',
config=True,
help="""The command to start the http proxy.
Only override if configurable-http-proxy is not on your PATH
""")
debug_proxy = Bool(False,
config=True,
help="show debug output in configurable-http-proxy")
proxy_auth_token = Unicode(config=True,
help="""The Proxy Auth token.
Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
""")
def _proxy_auth_token_default(self):
token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
if not token:
self.log.warn('\n'.join([
"",
"Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.",
"Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.",
"",
]))
token = orm.new_token()
return token
proxy_api_ip = Unicode('localhost',
config=True,
help="The ip for the proxy API handlers")
proxy_api_port = Integer(config=True,
help="The port for the proxy API handlers")
def _proxy_api_port_default(self):
return self.port + 1
hub_port = Integer(8081, config=True, help="The port for this process")
hub_ip = Unicode('localhost', config=True, help="The ip for this process")
hub_prefix = URLPrefix(
'/hub/',
config=True,
help="The prefix for the hub server. Must not be '/'")
def _hub_prefix_default(self):
return url_path_join(self.base_url, '/hub/')
def _hub_prefix_changed(self, name, old, new):
if new == '/':
raise TraitError("'/' is not a valid hub prefix")
if not new.startswith(self.base_url):
self.hub_prefix = url_path_join(self.base_url, new)
cookie_secret = Bytes(config=True,
env='JPY_COOKIE_SECRET',
help="""The cookie secret to use to encrypt cookies.
Loaded from the JPY_COOKIE_SECRET env variable by default.
""")
cookie_secret_file = Unicode(
'jupyterhub_cookie_secret',
config=True,
help="""File in which to store the cookie secret.""")
authenticator_class = Type(PAMAuthenticator,
Authenticator,
config=True,
help="""Class for authenticating users.
This should be a class with the following form:
- constructor takes one kwarg: `config`, the IPython config object.
- is a tornado.gen.coroutine
- returns username on success, None on failure
- takes two arguments: (handler, data),
where `handler` is the calling web.RequestHandler,
and `data` is the POST form data from the login page.
""")
authenticator = Instance(Authenticator)
def _authenticator_default(self):
return self.authenticator_class(parent=self, db=self.db)
# class for spawning single-user servers
spawner_class = Type(
LocalProcessSpawner,
Spawner,
config=True,
help="""The class to use for spawning single-user servers.
Should be a subclass of Spawner.
""")
db_url = Unicode(
'sqlite:///jupyterhub.sqlite',
config=True,
help="url for the database. e.g. `sqlite:///jupyterhub.sqlite`")
def _db_url_changed(self, name, old, new):
if '://' not in new:
# assume sqlite, if given as a plain filename
self.db_url = 'sqlite:///%s' % new
db_kwargs = Dict(
config=True,
help="""Include any kwargs to pass to the database connection.
See sqlalchemy.create_engine for details.
""")
reset_db = Bool(False, config=True, help="Purge and reset the database.")
debug_db = Bool(
False,
config=True,
help="log all database transactions. This has A LOT of output")
db = Any()
session_factory = Any()
admin_access = Bool(
False,
config=True,
help="""Grant admin users permission to access single-user servers.
Users should be properly informed if this is enabled.
""")
admin_users = Set(
config=True,
help="""DEPRECATED, use Authenticator.admin_users instead.""")
tornado_settings = Dict(config=True)
cleanup_servers = Bool(
True,
config=True,
help="""Whether to shutdown single-user servers when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the single-user servers running.
If both this and cleanup_proxy are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
""")
cleanup_proxy = Bool(
True,
config=True,
help="""Whether to shutdown the proxy when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the proxy running.
Only valid if the proxy was starting by the Hub process.
If both this and cleanup_servers are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
""")
handlers = List()
_log_formatter_cls = CoroutineLogFormatter
http_server = None
proxy_process = None
io_loop = None
def _log_level_default(self):
return logging.INFO
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
extra_log_file = Unicode("",
config=True,
help="Set a logging.FileHandler on this file.")
extra_log_handlers = List(
Instance(logging.Handler),
config=True,
help="Extra log handlers to set on JupyterHub logger",
)
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
if self.extra_log_file:
self.extra_log_handlers.append(
logging.FileHandler(self.extra_log_file))
_formatter = self._log_formatter_cls(
fmt=self.log_format,
datefmt=self.log_datefmt,
)
for handler in self.extra_log_handlers:
if handler.formatter is None:
handler.setFormatter(_formatter)
self.log.addHandler(handler)
# hook up tornado 3's loggers to our app handlers
for log in (app_log, access_log, gen_log):
# ensure all log statements identify the application they come from
log.name = self.log.name
logger = logging.getLogger('tornado')
logger.propagate = True
logger.parent = self.log
logger.setLevel(self.log.level)
def init_ports(self):
if self.hub_port == self.port:
raise TraitError(
"The hub and proxy cannot both listen on port %i" % self.port)
if self.hub_port == self.proxy_api_port:
raise TraitError(
"The hub and proxy API cannot both listen on port %i" %
self.hub_port)
if self.proxy_api_port == self.port:
raise TraitError(
"The proxy's public and API ports cannot both be %i" %
self.port)
@staticmethod
def add_url_prefix(prefix, handlers):
"""add a url prefix to handlers"""
for i, tup in enumerate(handlers):
lis = list(tup)
lis[0] = url_path_join(prefix, tup[0])
handlers[i] = tuple(lis)
return handlers
def init_handlers(self):
h = []
h.extend(handlers.default_handlers)
h.extend(apihandlers.default_handlers)
# load handlers from the authenticator
h.extend(self.authenticator.get_handlers(self))
self.handlers = self.add_url_prefix(self.hub_prefix, h)
# some extra handlers, outside hub_prefix
self.handlers.extend([
(r"%s" % self.hub_prefix.rstrip('/'), web.RedirectHandler, {
"url": self.hub_prefix,
"permanent": False,
}),
(r"(?!%s).*" % self.hub_prefix, handlers.PrefixRedirectHandler),
(r'(.*)', handlers.Template404),
])
def _check_db_path(self, path):
"""More informative log messages for failed filesystem access"""
path = os.path.abspath(path)
parent, fname = os.path.split(path)
user = getuser()
if not os.path.isdir(parent):
self.log.error("Directory %s does not exist", parent)
if os.path.exists(parent) and not os.access(parent, os.W_OK):
self.log.error("%s cannot create files in %s", user, parent)
if os.path.exists(path) and not os.access(path, os.W_OK):
self.log.error("%s cannot edit %s", user, path)
def init_secrets(self):
trait_name = 'cookie_secret'
trait = self.traits()[trait_name]
env_name = trait.get_metadata('env')
secret_file = os.path.abspath(
os.path.expanduser(self.cookie_secret_file))
secret = self.cookie_secret
secret_from = 'config'
# load priority: 1. config, 2. env, 3. file
if not secret and os.environ.get(env_name):
secret_from = 'env'
self.log.info("Loading %s from env[%s]", trait_name, env_name)
secret = binascii.a2b_hex(os.environ[env_name])
if not secret and os.path.exists(secret_file):
secret_from = 'file'
perm = os.stat(secret_file).st_mode
if perm & 0o077:
self.log.error("Bad permissions on %s", secret_file)
else:
self.log.info("Loading %s from %s", trait_name, secret_file)
with open(secret_file) as f:
b64_secret = f.read()
try:
secret = binascii.a2b_base64(b64_secret)
except Exception as e:
self.log.error("%s does not contain b64 key: %s",
secret_file, e)
if not secret:
secret_from = 'new'
self.log.debug("Generating new %s", trait_name)
secret = os.urandom(SECRET_BYTES)
if secret_file and secret_from == 'new':
# if we generated a new secret, store it in the secret_file
self.log.info("Writing %s to %s", trait_name, secret_file)
b64_secret = binascii.b2a_base64(secret).decode('ascii')
with open(secret_file, 'w') as f:
f.write(b64_secret)
try:
os.chmod(secret_file, 0o600)
except OSError:
self.log.warn("Failed to set permissions on %s", secret_file)
# store the loaded trait value
self.cookie_secret = secret
def init_db(self):
"""Create the database connection"""
self.log.debug("Connecting to db: %s", self.db_url)
try:
self.session_factory = orm.new_session_factory(self.db_url,
reset=self.reset_db,
echo=self.debug_db,
**self.db_kwargs)
self.db = scoped_session(self.session_factory)()
except OperationalError as e:
self.log.error("Failed to connect to db: %s", self.db_url)
self.log.debug("Database error was:", exc_info=True)
if self.db_url.startswith('sqlite:///'):
self._check_db_path(self.db_url.split(':///', 1)[1])
self.exit(1)
def init_hub(self):
"""Load the Hub config into the database"""
self.hub = self.db.query(orm.Hub).first()
if self.hub is None:
self.hub = orm.Hub(server=orm.Server(
ip=self.hub_ip,
port=self.hub_port,
base_url=self.hub_prefix,
cookie_name='jupyter-hub-token',
))
self.db.add(self.hub)
else:
server = self.hub.server
server.ip = self.hub_ip
server.port = self.hub_port
server.base_url = self.hub_prefix
self.db.commit()
@gen.coroutine
def init_users(self):
"""Load users into and from the database"""
db = self.db
if self.admin_users and not self.authenticator.admin_users:
self.log.warn("\nJupyterHub.admin_users is deprecated."
"\nUse Authenticator.admin_users instead.")
self.authenticator.admin_users = self.admin_users
admin_users = self.authenticator.admin_users
if not admin_users:
# add current user as admin if there aren't any others
admins = db.query(orm.User).filter(orm.User.admin == True)
if admins.first() is None:
admin_users.add(getuser())
new_users = []
for name in admin_users:
# ensure anyone specified as admin in config is admin in db
user = orm.User.find(db, name)
if user is None:
user = orm.User(name=name, admin=True)
new_users.append(user)
db.add(user)
else:
user.admin = True
# the admin_users config variable will never be used after this point.
# only the database values will be referenced.
whitelist = self.authenticator.whitelist
if not whitelist:
self.log.info(
"Not using whitelist. Any authenticated user will be allowed.")
# add whitelisted users to the db
for name in whitelist:
user = orm.User.find(db, name)
if user is None:
user = orm.User(name=name)
new_users.append(user)
db.add(user)
if whitelist:
# fill the whitelist with any users loaded from the db,
# so we are consistent in both directions.
# This lets whitelist be used to set up initial list,
# but changes to the whitelist can occur in the database,
# and persist across sessions.
for user in db.query(orm.User):
whitelist.add(user.name)
# The whitelist set and the users in the db are now the same.
# From this point on, any user changes should be done simultaneously
# to the whitelist set and user db, unless the whitelist is empty (all users allowed).
db.commit()
for user in new_users:
yield gen.maybe_future(self.authenticator.add_user(user))
db.commit()
@gen.coroutine
def init_spawners(self):
db = self.db
user_summaries = ['']
def _user_summary(user):
parts = ['{0: >8}'.format(user.name)]
if user.admin:
parts.append('admin')
if user.server:
parts.append('running at %s' % user.server)
return ' '.join(parts)
@gen.coroutine
def user_stopped(user):
status = yield user.spawner.poll()
self.log.warn(
"User %s server stopped with exit code: %s",
user.name,
status,
)
yield self.proxy.delete_user(user)
yield user.stop()
for user in db.query(orm.User):
if not user.state:
# without spawner state, server isn't valid
user.server = None
user_summaries.append(_user_summary(user))
continue
self.log.debug("Loading state for %s from db", user.name)
user.spawner = spawner = self.spawner_class(
user=user,
hub=self.hub,
config=self.config,
db=self.db,
)
status = yield spawner.poll()
if status is None:
self.log.info("%s still running", user.name)
spawner.add_poll_callback(user_stopped, user)
spawner.start_polling()
else:
# user not running. This is expected if server is None,
# but indicates the user's server died while the Hub wasn't running
# if user.server is defined.
log = self.log.warn if user.server else self.log.debug
log("%s not running.", user.name)
user.server = None
user_summaries.append(_user_summary(user))
self.log.debug("Loaded users: %s", '\n'.join(user_summaries))
db.commit()
def init_proxy(self):
"""Load the Proxy config into the database"""
self.proxy = self.db.query(orm.Proxy).first()
if self.proxy is None:
self.proxy = orm.Proxy(
public_server=orm.Server(),
api_server=orm.Server(),
)
self.db.add(self.proxy)
self.db.commit()
self.proxy.auth_token = self.proxy_auth_token # not persisted
self.proxy.log = self.log
self.proxy.public_server.ip = self.ip
self.proxy.public_server.port = self.port
self.proxy.api_server.ip = self.proxy_api_ip
self.proxy.api_server.port = self.proxy_api_port
self.proxy.api_server.base_url = '/api/routes/'
self.db.commit()
@gen.coroutine
def start_proxy(self):
"""Actually start the configurable-http-proxy"""
# check for proxy
if self.proxy.public_server.is_up() or self.proxy.api_server.is_up():
# check for *authenticated* access to the proxy (auth token can change)
try:
yield self.proxy.get_routes()
except (HTTPError, OSError, socket.error) as e:
if isinstance(e, HTTPError) and e.code == 403:
msg = "Did CONFIGPROXY_AUTH_TOKEN change?"
else:
msg = "Is something else using %s?" % self.proxy.public_server.bind_url
self.log.error(
"Proxy appears to be running at %s, but I can't access it (%s)\n%s",
self.proxy.public_server.bind_url, e, msg)
self.exit(1)
return
else:
self.log.info("Proxy already running at: %s",
self.proxy.public_server.bind_url)
self.proxy_process = None
return
env = os.environ.copy()
env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token
cmd = self.proxy_cmd + [
'--ip',
self.proxy.public_server.ip,
'--port',
str(self.proxy.public_server.port),
'--api-ip',
self.proxy.api_server.ip,
'--api-port',
str(self.proxy.api_server.port),
'--default-target',
self.hub.server.host,
]
if self.debug_proxy:
cmd.extend(['--log-level', 'debug'])
if self.ssl_key:
cmd.extend(['--ssl-key', self.ssl_key])
if self.ssl_cert:
cmd.extend(['--ssl-cert', self.ssl_cert])
self.log.info("Starting proxy @ %s", self.proxy.public_server.bind_url)
self.log.debug("Proxy cmd: %s", cmd)
try:
self.proxy_process = Popen(cmd, env=env)
except FileNotFoundError as e:
self.log.error(
"Failed to find proxy %r\n"
"The proxy can be installed with `npm install -g configurable-http-proxy`"
% self.proxy_cmd)
self.exit(1)
def _check():
status = self.proxy_process.poll()
if status is not None:
e = RuntimeError("Proxy failed to start with exit code %i" %
status)
# py2-compatible `raise e from None`
e.__cause__ = None
raise e
for server in (self.proxy.public_server, self.proxy.api_server):
for i in range(10):
_check()
try:
yield server.wait_up(1)
except TimeoutError:
continue
else:
break
yield server.wait_up(1)
self.log.debug("Proxy started and appears to be up")
@gen.coroutine
def check_proxy(self):
if self.proxy_process.poll() is None:
return
self.log.error(
"Proxy stopped with exit code %r", 'unknown'
if self.proxy_process is None else self.proxy_process.poll())
yield self.start_proxy()
self.log.info("Setting up routes on new proxy")
yield self.proxy.add_all_users()
self.log.info("New proxy back up, and good to go")
def init_tornado_settings(self):
"""Set up the tornado settings dict."""
base_url = self.hub.server.base_url
jinja_env = Environment(loader=FileSystemLoader(self.template_paths),
**self.jinja_environment_options)
login_url = self.authenticator.login_url(base_url)
logout_url = self.authenticator.logout_url(base_url)
# if running from git, disable caching of require.js
# otherwise cache based on server start time
parent = os.path.dirname(os.path.dirname(jupyterhub.__file__))
if os.path.isdir(os.path.join(parent, '.git')):
version_hash = ''
else:
version_hash = datetime.now().strftime("%Y%m%d%H%M%S"),
settings = dict(
log_function=log_request,
config=self.config,
log=self.log,
db=self.db,
proxy=self.proxy,
hub=self.hub,
admin_users=self.authenticator.admin_users,
admin_access=self.admin_access,
authenticator=self.authenticator,
spawner_class=self.spawner_class,
base_url=self.base_url,
cookie_secret=self.cookie_secret,
cookie_max_age_days=self.cookie_max_age_days,
login_url=login_url,
logout_url=logout_url,
static_path=os.path.join(self.data_files_path, 'static'),
static_url_prefix=url_path_join(self.hub.server.base_url,
'static/'),
static_handler_class=CacheControlStaticFilesHandler,
template_path=self.template_paths,
jinja2_env=jinja_env,
version_hash=version_hash,
)
# allow configured settings to have priority
settings.update(self.tornado_settings)
self.tornado_settings = settings
def init_tornado_application(self):
"""Instantiate the tornado Application object"""
self.tornado_application = web.Application(self.handlers,
**self.tornado_settings)
def write_pid_file(self):
pid = os.getpid()
if self.pid_file:
self.log.debug("Writing PID %i to %s", pid, self.pid_file)
with open(self.pid_file, 'w') as f:
f.write('%i' % pid)
@gen.coroutine
@catch_config_error
def initialize(self, *args, **kwargs):
super().initialize(*args, **kwargs)
if self.generate_config or self.subapp:
return
self.load_config_file(self.config_file)
self.init_logging()
if 'JupyterHubApp' in self.config:
self.log.warn(
"Use JupyterHub in config, not JupyterHubApp. Outdated config:\n%s",
'\n'.join('JupyterHubApp.{key} = {value!r}'.format(key=key,
value=value)
for key, value in self.config.JupyterHubApp.items()))
cfg = self.config.copy()
cfg.JupyterHub.merge(cfg.JupyterHubApp)
self.update_config(cfg)
self.write_pid_file()
self.init_ports()
self.init_secrets()
self.init_db()
self.init_hub()
self.init_proxy()
yield self.init_users()
yield self.init_spawners()
self.init_handlers()
self.init_tornado_settings()
self.init_tornado_application()
@gen.coroutine
def cleanup(self):
"""Shutdown our various subprocesses and cleanup runtime files."""
futures = []
if self.cleanup_servers:
self.log.info("Cleaning up single-user servers...")
# request (async) process termination
for user in self.db.query(orm.User):
if user.spawner is not None:
futures.append(user.stop())
else:
self.log.info("Leaving single-user servers running")
# clean up proxy while SUS are shutting down
if self.cleanup_proxy:
if self.proxy_process:
self.log.info("Cleaning up proxy[%i]...",
self.proxy_process.pid)
if self.proxy_process.poll() is None:
try:
self.proxy_process.terminate()
except Exception as e:
self.log.error("Failed to terminate proxy process: %s",
e)
else:
self.log.info("I didn't start the proxy, I can't clean it up")
else:
self.log.info("Leaving proxy running")
# wait for the requests to stop finish:
for f in futures:
try:
yield f
except Exception as e:
self.log.error("Failed to stop user: %s", e)
self.db.commit()
if self.pid_file and os.path.exists(self.pid_file):
self.log.info("Cleaning up PID file %s", self.pid_file)
os.remove(self.pid_file)
# finally stop the loop once we are all cleaned up
self.log.info("...done")
def write_config_file(self):
"""Write our default config to a .py config file"""
if os.path.exists(self.config_file) and not self.answer_yes:
answer = ''
def ask():
prompt = "Overwrite %s with default config? [y/N]" % self.config_file
try:
return input(prompt).lower() or 'n'
except KeyboardInterrupt:
print('') # empty line
return 'n'
answer = ask()
while not answer.startswith(('y', 'n')):
print("Please answer 'yes' or 'no'")
answer = ask()
if answer.startswith('n'):
return
config_text = self.generate_config_file()
if isinstance(config_text, bytes):
config_text = config_text.decode('utf8')
print("Writing default config to: %s" % self.config_file)
with open(self.config_file, mode='w') as f:
f.write(config_text)
@gen.coroutine
def update_last_activity(self):
"""Update User.last_activity timestamps from the proxy"""
routes = yield self.proxy.get_routes()
for prefix, route in routes.items():
if 'user' not in route:
# not a user route, ignore it
continue
user = orm.User.find(self.db, route['user'])
if user is None:
self.log.warn("Found no user for route: %s", route)
continue
try:
dt = datetime.strptime(route['last_activity'], ISO8601_ms)
except Exception:
dt = datetime.strptime(route['last_activity'], ISO8601_s)
user.last_activity = max(user.last_activity, dt)
self.db.commit()
yield self.proxy.check_routes(routes)
@gen.coroutine
def start(self):
"""Start the whole thing"""
self.io_loop = loop = IOLoop.current()
if self.subapp:
self.subapp.start()
loop.stop()
return
if self.generate_config:
self.write_config_file()
loop.stop()
return
# start the webserver
self.http_server = tornado.httpserver.HTTPServer(
self.tornado_application, xheaders=True)
try:
self.http_server.listen(self.hub_port, address=self.hub_ip)
except Exception:
self.log.error("Failed to bind hub to %s",
self.hub.server.bind_url)
raise
else:
self.log.info("Hub API listening on %s", self.hub.server.bind_url)
# start the proxy
try:
yield self.start_proxy()
except Exception as e:
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
return
loop.add_callback(self.proxy.add_all_users)
if self.proxy_process:
# only check / restart the proxy if we started it in the first place.
# this means a restarted Hub cannot restart a Proxy that its
# predecessor started.
pc = PeriodicCallback(self.check_proxy,
1e3 * self.proxy_check_interval)
pc.start()
if self.last_activity_interval:
pc = PeriodicCallback(self.update_last_activity,
1e3 * self.last_activity_interval)
pc.start()
self.log.info("JupyterHub is now running at %s",
self.proxy.public_server.url)
# register cleanup on both TERM and INT
atexit.register(self.atexit)
self.init_signal()
def init_signal(self):
signal.signal(signal.SIGTERM, self.sigterm)
def sigterm(self, signum, frame):
self.log.critical("Received SIGTERM, shutting down")
self.io_loop.stop()
self.atexit()
_atexit_ran = False
def atexit(self):
"""atexit callback"""
if self._atexit_ran:
return
self._atexit_ran = True
# run the cleanup step (in a new loop, because the interrupted one is unclean)
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self.cleanup)
def stop(self):
if not self.io_loop:
return
if self.http_server:
if self.io_loop._running:
self.io_loop.add_callback(self.http_server.stop)
else:
self.http_server.stop()
self.io_loop.add_callback(self.io_loop.stop)
@gen.coroutine
def launch_instance_async(self, argv=None):
try:
yield self.initialize(argv)
yield self.start()
except Exception as e:
self.log.exception("")
self.exit(1)
@classmethod
def launch_instance(cls, argv=None):
self = cls.instance()
loop = IOLoop.current()
loop.add_callback(self.launch_instance_async, argv)
try:
loop.start()
except KeyboardInterrupt:
print("\nInterrupted")
class FileContentsManager(FileManagerMixin, ContentsManager):
root_dir = Unicode(config=True)
@default('root_dir')
def _default_root_dir(self):
try:
return self.parent.notebook_dir
except AttributeError:
return getcwd()
save_script = Bool(
False,
config=True,
help='DEPRECATED, use post_save_hook. Will be removed in Notebook 5.0')
@observe('save_script')
def _update_save_script(self, change):
if not change['new']:
return
self.log.warning("""
`--script` is deprecated and will be removed in notebook 5.0.
You can trigger nbconvert via pre- or post-save hooks:
ContentsManager.pre_save_hook
FileContentsManager.post_save_hook
A post-save hook has been registered that calls:
jupyter nbconvert --to script [notebook]
which behaves similarly to `--script`.
""")
self.post_save_hook = _post_save_script
post_save_hook = Any(None,
config=True,
allow_none=True,
help="""Python callable or importstring thereof
to be called on the path of a file just saved.
This can be used to process the file on disk,
such as converting the notebook to a script or HTML via nbconvert.
It will be called as (all arguments passed by keyword)::
hook(os_path=os_path, model=model, contents_manager=instance)
- path: the filesystem path to the file just written
- model: the model representing the file
- contents_manager: this ContentsManager instance
""")
@validate('post_save_hook')
def _validate_post_save_hook(self, proposal):
value = proposal['value']
if isinstance(value, string_types):
value = import_item(value)
if not callable(value):
raise TraitError("post_save_hook must be callable")
return value
def run_post_save_hook(self, model, os_path):
"""Run the post-save hook if defined, and log errors"""
if self.post_save_hook:
try:
self.log.debug("Running post-save hook on %s", os_path)
self.post_save_hook(os_path=os_path,
model=model,
contents_manager=self)
except Exception as e:
self.log.error("Post-save hook failed o-n %s",
os_path,
exc_info=True)
raise web.HTTPError(
500,
u'Unexpected error while running post hook save: %s' % e)
@validate('root_dir')
def _validate_root_dir(self, proposal):
"""Do a bit of validation of the root_dir."""
value = proposal['value']
if not os.path.isabs(value):
# If we receive a non-absolute path, make it absolute.
value = os.path.abspath(value)
if not os.path.isdir(value):
raise TraitError("%r is not a directory" % value)
return value
@default('checkpoints_class')
def _checkpoints_class_default(self):
return FileCheckpoints
delete_to_trash = Bool(
True,
config=True,
help="""If True (default), deleting files will send them to the
platform's trash/recycle bin, where they can be recovered. If False,
deleting files really deletes them.""")
@default('files_handler_class')
def _files_handler_class_default(self):
return AuthenticatedFileHandler
@default('files_handler_params')
def _files_handler_params_default(self):
return {'path': self.root_dir}
def is_hidden(self, path):
"""Does the API style path correspond to a hidden directory or file?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
hidden : bool
Whether the path exists and is hidden.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return is_hidden(os_path, self.root_dir)
def file_exists(self, path):
"""Returns True if the file exists, else returns False.
API-style wrapper for os.path.isfile
Parameters
----------
path : string
The relative path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the file exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path)
return os.path.isfile(os_path)
def dir_exists(self, path):
"""Does the API-style path refer to an extant directory?
API-style wrapper for os.path.isdir
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
exists : bool
Whether the path is indeed a directory.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.isdir(os_path)
def exists(self, path):
"""Returns True if the path exists, else returns False.
API-style wrapper for os.path.exists
Parameters
----------
path : string
The API path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the target exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return exists(os_path)
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.lstat(os_path)
try:
last_modified = tz.utcfromtimestamp(info.st_mtime)
except (ValueError, OSError):
# Files can rarely have an invalid timestamp
# https://github.com/jupyter/notebook/issues/2539
# https://github.com/jupyter/notebook/issues/2757
# Use the Unix epoch as a fallback so we don't crash.
self.log.warning('Invalid mtime %s for %s', info.st_mtime, os_path)
last_modified = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
try:
created = tz.utcfromtimestamp(info.st_ctime)
except (ValueError, OSError): # See above
self.log.warning('Invalid ctime %s for %s', info.st_ctime, os_path)
created = datetime(1970, 1, 1, 0, 0, tzinfo=tz.UTC)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
os_path = self._get_os_path(path)
four_o_four = u'directory does not exist: %r' % path
if not os.path.isdir(os_path):
raise web.HTTPError(404, four_o_four)
elif is_hidden(os_path, self.root_dir) and not self.allow_hidden:
self.log.info(
"Refusing to serve hidden directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
if content:
model['content'] = contents = []
os_dir = self._get_os_path(path)
for name in os.listdir(os_dir):
try:
os_path = os.path.join(os_dir, name)
except UnicodeDecodeError as e:
self.log.warning("failed to decode filename '%s': %s",
name, e)
continue
try:
st = os.lstat(os_path)
except OSError as e:
# skip over broken symlinks in listing
if e.errno == errno.ENOENT:
self.log.warning("%s doesn't exist", os_path)
else:
self.log.warning("Error stat-ing %s: %s", os_path, e)
continue
if (not stat.S_ISLNK(st.st_mode)
and not stat.S_ISREG(st.st_mode)
and not stat.S_ISDIR(st.st_mode)):
self.log.debug("%s not a regular file", os_path)
continue
if self.should_list(name) and not is_file_hidden(os_path,
stat_res=st):
contents.append(
self.get(path='%s/%s' % (path, name), content=False))
model['format'] = 'json'
return model
def _file_model(self, path, content=True, format=None):
"""Build a model for a file
if content is requested, include the file contents.
format:
If 'text', the contents will be decoded as UTF-8.
If 'base64', the raw bytes contents will be encoded as base64.
If not specified, try to decode as UTF-8, and fall back to base64
"""
model = self._base_model(path)
model['type'] = 'file'
os_path = self._get_os_path(path)
model['mimetype'] = mimetypes.guess_type(os_path)[0]
if content:
content, format = self._read_file(os_path, format)
if model['mimetype'] is None:
default_mime = {
'text': 'text/plain',
'base64': 'application/octet-stream'
}[format]
model['mimetype'] = default_mime
model.update(
content=content,
format=format,
)
return model
def _notebook_model(self, path, content=True):
"""Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
model = self._base_model(path)
model['type'] = 'notebook'
if content:
os_path = self._get_os_path(path)
nb = self._read_notebook(os_path, as_version=4)
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def get(self, path, content=True, type=None, format=None):
""" Takes a path for an entity and returns its model
Parameters
----------
path : str
the API path that describes the relative path for the target
content : bool
Whether to include the contents in the reply
type : str, optional
The requested type - 'file', 'notebook', or 'directory'.
Will raise HTTPError 400 if the content doesn't match.
format : str, optional
The requested format for file contents. 'text' or 'base64'.
Ignored if this returns a notebook or directory model.
Returns
-------
model : dict
the contents model. If content=True, returns the contents
of the file or directory as well.
"""
path = path.strip('/')
if not self.exists(path):
raise web.HTTPError(404, u'No such file or directory: %s' % path)
os_path = self._get_os_path(path)
if os.path.isdir(os_path):
if type not in (None, 'directory'):
raise web.HTTPError(400,
u'%s is a directory, not a %s' %
(path, type),
reason='bad type')
model = self._dir_model(path, content=content)
elif type == 'notebook' or (type is None and path.endswith('.ipynb')):
model = self._notebook_model(path, content=content)
else:
if type == 'directory':
raise web.HTTPError(400,
u'%s is not a directory' % path,
reason='bad type')
model = self._file_model(path, content=content, format=format)
return model
def _save_directory(self, os_path, model, path=''):
"""create a directory"""
if is_hidden(os_path, self.root_dir) and not self.allow_hidden:
raise web.HTTPError(400,
u'Cannot create hidden directory %r' % os_path)
if not os.path.exists(os_path):
with self.perm_to_403():
os.mkdir(os_path)
elif not os.path.isdir(os_path):
raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
else:
self.log.debug("Directory %r already exists", os_path)
def save(self, model, path=''):
"""Save the file model and return the model with no content."""
path = path.strip('/')
if 'type' not in model:
raise web.HTTPError(400, u'No file type provided')
if 'content' not in model and model['type'] != 'directory':
raise web.HTTPError(400, u'No file content provided')
os_path = self._get_os_path(path)
self.log.debug("Saving %s", os_path)
self.run_pre_save_hook(model=model, path=path)
try:
if model['type'] == 'notebook':
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
self._save_notebook(os_path, nb)
# One checkpoint should always exist for notebooks.
if not self.checkpoints.list_checkpoints(path):
self.create_checkpoint(path)
elif model['type'] == 'file':
# Missing format will be handled internally by _save_file.
self._save_file(os_path, model['content'], model.get('format'))
elif model['type'] == 'directory':
self._save_directory(os_path, model, path)
else:
raise web.HTTPError(
400, "Unhandled contents type: %s" % model['type'])
except web.HTTPError:
raise
except Exception as e:
self.log.error(u'Error while saving file: %s %s',
path,
e,
exc_info=True)
raise web.HTTPError(
500, u'Unexpected error while saving file: %s %s' % (path, e))
validation_message = None
if model['type'] == 'notebook':
self.validate_notebook_model(model)
validation_message = model.get('message', None)
model = self.get(path, content=False)
if validation_message:
model['message'] = validation_message
self.run_post_save_hook(model=model, os_path=os_path)
return model
def delete_file(self, path):
"""Delete file at path."""
path = path.strip('/')
os_path = self._get_os_path(path)
rm = os.unlink
if not os.path.exists(os_path):
raise web.HTTPError(
404, u'File or directory does not exist: %s' % os_path)
if self.delete_to_trash:
self.log.debug("Sending %s to trash", os_path)
# Looking at the code in send2trash, I don't think the errors it
# raises let us distinguish permission errors from other errors in
# code. So for now, just let them all get logged as server errors.
send2trash(os_path)
return
if os.path.isdir(os_path):
listing = os.listdir(os_path)
# Don't permanently delete non-empty directories.
# A directory containing only leftover checkpoints is
# considered empty.
cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None)
for entry in listing:
if entry != cp_dir:
raise web.HTTPError(400,
u'Directory %s not empty' % os_path)
self.log.debug("Removing directory %s", os_path)
with self.perm_to_403():
shutil.rmtree(os_path)
else:
self.log.debug("Unlinking file %s", os_path)
with self.perm_to_403():
rm(os_path)
def rename_file(self, old_path, new_path):
"""Rename a file."""
old_path = old_path.strip('/')
new_path = new_path.strip('/')
if new_path == old_path:
return
new_os_path = self._get_os_path(new_path)
old_os_path = self._get_os_path(old_path)
# Should we proceed with the move?
if os.path.exists(new_os_path) and not samefile(
old_os_path, new_os_path):
raise web.HTTPError(409, u'File already exists: %s' % new_path)
# Move the file
try:
with self.perm_to_403():
shutil.move(old_os_path, new_os_path)
except web.HTTPError:
raise
except Exception as e:
raise web.HTTPError(
500, u'Unknown error renaming file: %s %s' % (old_path, e))
def info_string(self):
return _("Serving notebooks from local directory: %s") % self.root_dir
def get_kernel_path(self, path, model=None):
"""Return the initial API path of a kernel associated with a given notebook"""
if self.dir_exists(path):
return path
if '/' in path:
parent_dir = path.rsplit('/', 1)[0]
else:
parent_dir = ''
return parent_dir
class ZMQDisplayPublisher(DisplayPublisher):
"""A display publisher that publishes data using a ZeroMQ PUB socket."""
session = Instance(Session, allow_none=True)
pub_socket = Any(allow_none=True)
parent_header = Dict({})
topic = CBytes(b'display_data')
# thread_local:
# An attribute used to ensure the correct output message
#Â is processed. See ibpykernel Issue 113 for a discussion.
_thread_local = Any()
def set_parent(self, parent):
"""Set the parent for outbound messages."""
self.parent_header = extract_header(parent)
def _flush_streams(self):
"""flush IO Streams prior to display"""
sys.stdout.flush()
sys.stderr.flush()
@default('_thread_local')
def _default_thread_local(self):
"""Initialize our thread local storage"""
return local()
@property
def _hooks(self):
if not hasattr(self._thread_local, 'hooks'):
# create new list for a new thread
self._thread_local.hooks = []
return self._thread_local.hooks
def publish(
self,
data,
metadata=None,
source=None,
transient=None,
update=False,
):
"""Publish a display-data message
Parameters
----------
data: dict
A mime-bundle dict, keyed by mime-type.
metadata: dict, optional
Metadata associated with the data.
transient: dict, optional, keyword-only
Transient data that may only be relevant during a live display,
such as display_id.
Transient data should not be persisted to documents.
update: bool, optional, keyword-only
If True, send an update_display_data message instead of display_data.
"""
self._flush_streams()
if metadata is None:
metadata = {}
if transient is None:
transient = {}
self._validate_data(data, metadata)
content = {}
content['data'] = encode_images(data)
content['metadata'] = metadata
content['transient'] = transient
msg_type = 'update_display_data' if update else 'display_data'
# Use 2-stage process to send a message,
# in order to put it through the transform
# hooks before potentially sending.
msg = self.session.msg(msg_type,
json_clean(content),
parent=self.parent_header)
# Each transform either returns a new
# message or None. If None is returned,
# the message has been 'used' and we return.
for hook in self._hooks:
msg = hook(msg)
if msg is None:
return
self.session.send(
self.pub_socket,
msg,
ident=self.topic,
)
def clear_output(self, wait=False):
"""Clear output associated with the current execution (cell).
Parameters
----------
wait: bool (default: False)
If True, the output will not be cleared immediately,
instead waiting for the next display before clearing.
This reduces bounce during repeated clear & display loops.
"""
content = dict(wait=wait)
self._flush_streams()
self.session.send(
self.pub_socket,
u'clear_output',
content,
parent=self.parent_header,
ident=self.topic,
)
def register_hook(self, hook):
"""
Registers a hook with the thread-local storage.
Parameters
----------
hook : Any callable object
Returns
-------
Either a publishable message, or `None`.
The DisplayHook objects must return a message from
the __call__ method if they still require the
`session.send` method to be called after tranformation.
Returning `None` will halt that execution path, and
session.send will not be called.
"""
self._hooks.append(hook)
def unregister_hook(self, hook):
"""
Un-registers a hook with the thread-local storage.
Parameters
----------
hook: Any callable object which has previously been
registered as a hook.
Returns
-------
bool - `True` if the hook was removed, `False` if it wasn't
found.
"""
try:
self._hooks.remove(hook)
return True
except ValueError:
return False
class GFKernel(Kernel):
implementation = 'GF'
implementation_version = '1.0'
language = 'gf'
language_version = '0.1'
# TODO change this to gf
language_info = {
'name': 'gf',
'mimetype': 'text/gf',
'file_extension': '.gf',
}
banner = "GF"
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
shell_class = Type(ZMQInteractiveShell)
use_experimental_completions = Bool(
True,
help=
"Set this flag to False to deactivate the use of experimental IPython completion APIs.",
).tag(config=True)
user_module = Any()
def _user_module_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_module = new
user_ns = Instance(dict, args=None, allow_none=True)
def _user_ns_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_ns = new
self.shell.init_user_ns()
def __init__(self, **kwargs):
super(GFKernel, self).__init__(**kwargs)
# set up the shell
self.shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
# set up and attach comm_manager to the shell
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell.configurables.append(self.comm_manager)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager,
msg_type)
# initialize the GFRepl
self.GFRepl = GFRepl()
def start(self):
self.shell.exit_now = False
super(GFKernel, self).start()
def set_parent(self, ident, parent):
"""Overridden from parent to tell the display hook and output streams
about the parent message.
"""
super(GFKernel, self).set_parent(ident, parent)
self.shell.set_parent(parent)
def init_metadata(self, parent):
"""Initialize metadata.
Run at the beginning of each execution request.
"""
md = super(GFKernel, self).init_metadata(parent)
# FIXME: remove deprecated ipyparallel-specific code
# This is required for ipyparallel < 5.0
md.update({
'dependencies_met': True,
'engine': self.ident,
})
return md
def do_execute(self,
code,
silent=False,
store_history=True,
user_expressions=None,
allow_stdin=True):
"""Called when the user inputs code"""
# img_data = Image.open('/home/kai/gf_content/out.png','r')
messages = self.GFRepl.handle_input(code)
for msg in messages:
if msg['file']:
file_name = msg['file']
try:
with open(file_name, "rb") as f:
img = f.read()
display(widgets.Image(value=img, format='png'))
except:
self.send_response(
self.iopub_socket, 'display_data',
to_display_data("There is no tree to show!"))
elif msg['message']:
self.send_response(self.iopub_socket, 'display_data',
to_display_data(msg['message']))
elif msg['trees']:
dd = widgets.Dropdown(
layout={'width': 'max-content'},
options=msg['trees'],
value=msg['trees'][0],
description='Tree of:',
disabled=False,
)
file_name = self.GFRepl.handle_single_view(
"%s %s" % (msg['tree_type'], msg['trees'][0]))
with open(file_name, "rb") as f:
img = f.read()
image = widgets.Image(value=img, format='png')
def on_value_change(change):
file_name = self.GFRepl.handle_single_view(
"%s %s" % (msg['tree_type'], change['new']))
with open(file_name, "rb") as f:
img = f.read()
image.value = img
dd.observe(on_value_change, names='value')
display(dd, image)
return {
'status': 'ok',
# The base class increments the execution count
'payload': [],
'execution_count': self.execution_count,
'user_expressions': {},
}
def do_shutdown(self, restart):
"""Called when the kernel is terminated"""
self.GFRepl.do_shutdown()
def do_complete(self, code, cursorPos):
"""Autocompletion when the user presses tab"""
# load the shortcuts from the unicode-latex-map
last_word = get_current_word(code, cursorPos)
matches = get_matches(last_word)
if not last_word or not matches:
matches = None
return {
'matches': matches,
'cursor_start': cursorPos - len(last_word),
'cursor_end': cursorPos,
'metadata': {},
'status': 'ok'
}
class S3FS(GenericFS):
access_key_id = Unicode(help="S3/AWS access key ID",
allow_none=True,
default_value=None).tag(
config=True, env="JPYNB_S3_ACCESS_KEY_ID")
secret_access_key = Unicode(help="S3/AWS secret access key",
allow_none=True,
default_value=None).tag(
config=True,
env="JPYNB_S3_SECRET_ACCESS_KEY")
endpoint_url = Unicode("s3.amazonaws.com", help="S3 endpoint URL").tag(
config=True, env="JPYNB_S3_ENDPOINT_URL")
region_name = Unicode("us-east-1",
help="Region name").tag(config=True,
env="JPYNB_S3_REGION_NAME")
bucket = Unicode("notebooks", help="Bucket name to store notebooks").tag(
config=True, env="JPYNB_S3_BUCKET")
signature_version = Unicode(help="").tag(config=True)
sse = Unicode(help="Type of server-side encryption to use").tag(
config=True)
kms_key_id = Unicode(help="KMS ID to use to encrypt workbooks").tag(
config=True)
prefix = Unicode(
"", help="Prefix path inside the specified bucket").tag(config=True)
delimiter = Unicode("/", help="Path delimiter").tag(config=True)
dir_keep_file = Unicode(
".s3keep",
help="Empty file to create when creating directories").tag(config=True)
session_token = Unicode(help="S3/AWS session token",
allow_none=True,
default_value=None).tag(
config=True, env="JPYNB_S3_SESSION_TOKEN")
boto3_session = Any(
help="Place to store customer boto3 session instance - likely passed in"
)
def __init__(self, log, **kwargs):
super(S3FS, self).__init__(**kwargs)
self.log = log
client_kwargs = {
"endpoint_url": self.endpoint_url,
"region_name": self.region_name,
}
config_kwargs = {}
if self.signature_version:
config_kwargs["signature_version"] = self.signature_version
s3_additional_kwargs = {}
if self.sse:
s3_additional_kwargs["ServerSideEncryption"] = self.sse
if self.kms_key_id:
s3_additional_kwargs["SSEKMSKeyId"] = self.kms_key_id
self.fs = s3fs.S3FileSystem(
key=self.access_key_id,
secret=self.secret_access_key,
token=self.session_token,
client_kwargs=client_kwargs,
config_kwargs=config_kwargs,
s3_additional_kwargs=s3_additional_kwargs,
session=self.boto3_session,
)
self.init()
def init(self):
try:
self.mkdir("")
self.ls("")
self.isdir("")
except ClientError as ex:
if "AccessDenied" in str(ex):
policy = SAMPLE_ACCESS_POLICY.format(
bucket=os.path.join(self.bucket, self.prefix))
self.log.error(
"AccessDenied error while creating initial S3 objects. Create an IAM policy like:\n{policy}"
.format(policy=policy))
sys.exit(1)
else:
raise ex
# GenericFS methods -----------------------------------------------------------------------------------------------
def ls(self, path=""):
path_ = self.path(path)
self.log.debug("S3contents.S3FS: Listing directory: `%s`", path_)
files = self.fs.ls(path_, refresh=True)
return self.unprefix(files)
def isfile(self, path):
path_ = self.path(path)
# FileNotFoundError handled by s3fs
is_file = self.fs.isfile(path_)
self.log.debug("S3contents.S3FS: `%s` is a file: %s", path_, is_file)
return is_file
def isdir(self, path):
path_ = self.path(path)
# FileNotFoundError handled by s3fs
is_dir = self.fs.isdir(path_)
self.log.debug("S3contents.S3FS: `%s` is a directory: %s", path_,
is_dir)
return is_dir
def mv(self, old_path, new_path):
self.log.debug("S3contents.S3FS: Move file `%s` to `%s`", old_path,
new_path)
self.cp(old_path, new_path)
self.rm(old_path)
def cp(self, old_path, new_path):
old_path_, new_path_ = self.path(old_path), self.path(new_path)
self.log.debug("S3contents.S3FS: Coping `%s` to `%s`", old_path_,
new_path_)
if self.isdir(old_path):
old_dir_path, new_dir_path = old_path, new_path
for obj in self.ls(old_dir_path):
old_item_path = obj
new_item_path = old_item_path.replace(old_dir_path,
new_dir_path, 1)
self.cp(old_item_path, new_item_path)
self.mkdir(new_path) # Touch with dir_keep_file
elif self.isfile(old_path):
self.fs.copy(old_path_, new_path_)
def rm(self, path):
path_ = self.path(path)
self.log.debug("S3contents.S3FS: Removing: `%s`", path_)
if self.isfile(path):
self.log.debug("S3contents.S3FS: Removing file: `%s`", path_)
self.fs.rm(path_)
elif self.isdir(path):
self.log.debug("S3contents.S3FS: Removing directory: `%s`", path_)
self.fs.rm(path_ + self.delimiter, recursive=True)
# self.fs.rmdir(path_ + self.delimiter, recursive=True)
def mkdir(self, path):
path_ = self.path(path, self.dir_keep_file)
self.log.debug("S3contents.S3FS: Making dir: `%s`", path_)
self.fs.touch(path_)
def read(self, path, format):
path_ = self.path(path)
if not self.isfile(path):
raise NoSuchFile(path_)
with self.fs.open(path_, mode="rb") as f:
content = f.read()
if format is None or format == "text":
# Try to interpret as unicode if format is unknown or if unicode
# was explicitly requested.
try:
return content.decode("utf-8"), "text"
except UnicodeError:
if format == "text":
err = "{} is not UTF-8 encoded".format(path_)
self.log.error(err)
raise HTTPError(400, err, reason="bad format")
return base64.b64encode(content).decode("ascii"), "base64"
def lstat(self, path):
path_ = self.path(path)
if self.fs.isdir(path_): # Try to get status of the dir_keep_file
path_ = self.path(path, self.dir_keep_file)
try:
self.fs.invalidate_cache(path_)
info = self.fs.info(path_)
except FileNotFoundError:
return {"ST_MTIME": None}
ret = {}
ret["ST_MTIME"] = info["LastModified"]
return ret
def write(self, path, content, format):
path_ = self.path(self.unprefix(path))
self.log.debug("S3contents.S3FS: Writing file: `%s`", path_)
if format not in {"text", "base64"}:
raise HTTPError(
400,
"Must specify format of file contents as 'text' or 'base64'",
)
try:
if format == "text":
content_ = content.encode("utf8")
else:
b64_bytes = content.encode("ascii")
content_ = base64.b64decode(b64_bytes)
except Exception as e:
raise HTTPError(400, "Encoding error saving %s: %s" % (path_, e))
with self.fs.open(path_, mode="wb") as f:
f.write(content_)
def writenotebook(self, path, content):
path_ = self.path(self.unprefix(path))
self.log.debug("S3contents.S3FS: Writing notebook: `%s`", path_)
with self.fs.open(path_, mode="wb") as f:
f.write(content.encode("utf-8"))
# Utilities -------------------------------------------------------------------------------------------------------
def get_prefix(self):
"""Full prefix: bucket + optional prefix"""
prefix = self.bucket
if self.prefix:
prefix += self.delimiter + self.prefix
return prefix
prefix_ = property(get_prefix)
def unprefix(self, path):
"""Remove the self.prefix_ (if present) from a path or list of paths"""
if isinstance(path, str):
path = path[len(self.prefix_):] if path.startswith(
self.prefix_) else path
path = path[1:] if path.startswith(self.delimiter) else path
return path
if isinstance(path, (list, tuple)):
path = [
p[len(self.prefix_):] if p.startswith(self.prefix_) else p
for p in path
]
path = [p[1:] if p.startswith(self.delimiter) else p for p in path]
return path
def path(self, *path):
"""Utility to join paths including the bucket and prefix"""
path = list(filter(None, path))
path = self.unprefix(path)
items = [self.prefix_] + path
return self.delimiter.join(items)
class ImageCollectionBrowser(HasTraits):
paths = Union((List(), Unicode()))
file_index = Int(0)
path = Unicode()
images = Array()
image = Any()
frame_index = Int(0)
num_frames = Int(0)
binning = Int(1)
def __init__(self, **kwargs):
self._path_text = widgets.HTML(description='File name')
self._file_index_slider = IntSliderWithButtons(
description='File index', min=0)
self._frame_index_slider = IntSliderWithButtons(
description='Frame index', min=0)
link((self._file_index_slider, 'value'), (self, 'file_index'))
link((self._frame_index_slider, 'value'), (self, 'frame_index'))
super().__init__(**kwargs)
@observe('frame_index')
def _observe_frame_index(self, *args):
image = self.images[self.frame_index]
if self.binning != 1:
image = downscale_local_mean(image, factors=(self.binning, ) * 2)
self.image = image
@observe('path')
def _observe_path(self, *args):
self._path_text.value = str(Path(*Path(self.path).parts[-4:]))
images = imread(self.path)
# if images.shape[-1] in (3, 4):
# images = np.swapaxes(images, 0, 2)
if len(images.shape) == 2:
images = images[None]
assert len(images.shape) == 3
self.images = images
self.num_frames = len(self.images)
self._frame_index_slider.max = len(self.images) - 1
if self.frame_index == 0:
self._observe_frame_index()
else:
self.frame_index = 0
@observe('file_index')
def _observe_file_index(self, *args):
self.path = str(self.paths[self.file_index])
@observe('paths')
def _observe_filename(self, change):
self._file_index_slider.max = len(self.paths) - 1
if self.file_index == 0:
self._observe_file_index()
else:
self.file_index = 0
@property
def widgets(self):
hbox = widgets.VBox([
self._path_text, self._file_index_slider, self._frame_index_slider
])
return hbox
class ContentsManager(LoggingConfigurable):
"""Base class for serving files and directories.
This serves any text or binary file,
as well as directories,
with special handling for JSON notebook documents.
Most APIs take a path argument,
which is always an API-style unicode path,
and always refers to a directory.
- unicode, not url-escaped
- '/'-separated
- leading and trailing '/' will be stripped
- if unspecified, path defaults to '',
indicating the root path.
"""
root_dir = Unicode('/', config=True)
allow_hidden = Bool(False,
config=True,
help="Allow access to hidden files")
notary = Instance(sign.NotebookNotary)
def _notary_default(self):
return sign.NotebookNotary(parent=self)
hide_globs = List(Unicode(), [
u'__pycache__',
'*.pyc',
'*.pyo',
'.DS_Store',
'*.so',
'*.dylib',
'*~',
],
config=True,
help="""
Glob patterns to hide in file and directory listings.
""")
untitled_notebook = Unicode(
_("Untitled"),
config=True,
help="The base name used when creating untitled notebooks.")
untitled_file = Unicode(
"untitled",
config=True,
help="The base name used when creating untitled files.")
untitled_directory = Unicode(
"Untitled Folder",
config=True,
help="The base name used when creating untitled directories.")
pre_save_hook = Any(None,
config=True,
allow_none=True,
help="""Python callable or importstring thereof
To be called on a contents model prior to save.
This can be used to process the structure,
such as removing notebook outputs or other side effects that
should not be saved.
It will be called as (all arguments passed by keyword)::
hook(path=path, model=model, contents_manager=self)
- model: the model to be saved. Includes file contents.
Modifying this dict will affect the file that is stored.
- path: the API path of the save destination
- contents_manager: this ContentsManager instance
""")
@validate('pre_save_hook')
def _validate_pre_save_hook(self, proposal):
value = proposal['value']
if isinstance(value, string_types):
value = import_item(self.pre_save_hook)
if not callable(value):
raise TraitError("pre_save_hook must be callable")
return value
def run_pre_save_hook(self, model, path, **kwargs):
"""Run the pre-save hook if defined, and log errors"""
if self.pre_save_hook:
try:
self.log.debug("Running pre-save hook on %s", path)
self.pre_save_hook(model=model,
path=path,
contents_manager=self,
**kwargs)
except Exception:
self.log.error("Pre-save hook failed on %s",
path,
exc_info=True)
checkpoints_class = Type(Checkpoints, config=True)
checkpoints = Instance(Checkpoints, config=True)
checkpoints_kwargs = Dict(config=True)
@default('checkpoints')
def _default_checkpoints(self):
return self.checkpoints_class(**self.checkpoints_kwargs)
@default('checkpoints_kwargs')
def _default_checkpoints_kwargs(self):
return dict(
parent=self,
log=self.log,
)
files_handler_class = Type(
FilesHandler,
klass=RequestHandler,
allow_none=True,
config=True,
help="""handler class to use when serving raw file requests.
Default is a fallback that talks to the ContentsManager API,
which may be inefficient, especially for large files.
Local files-based ContentsManagers can use a StaticFileHandler subclass,
which will be much more efficient.
Access to these files should be Authenticated.
""")
files_handler_params = Dict(
config=True,
help="""Extra parameters to pass to files_handler_class.
For example, StaticFileHandlers generally expect a `path` argument
specifying the root directory from which to serve files.
""")
def get_extra_handlers(self):
"""Return additional handlers
Default: self.files_handler_class on /files/.*
"""
handlers = []
if self.files_handler_class:
handlers.append((r"/files/(.*)", self.files_handler_class,
self.files_handler_params))
return handlers
# ContentsManager API part 1: methods that must be
# implemented in subclasses.
def dir_exists(self, path):
"""Does a directory exist at the given path?
Like os.path.isdir
Override this method in subclasses.
Parameters
----------
path : string
The path to check
Returns
-------
exists : bool
Whether the path does indeed exist.
"""
raise NotImplementedError
def is_hidden(self, path):
"""Is path a hidden directory or file?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root dir).
Returns
-------
hidden : bool
Whether the path is hidden.
"""
raise NotImplementedError
def file_exists(self, path=''):
"""Does a file exist at the given path?
Like os.path.isfile
Override this method in subclasses.
Parameters
----------
path : string
The API path of a file to check for.
Returns
-------
exists : bool
Whether the file exists.
"""
raise NotImplementedError('must be implemented in a subclass')
def exists(self, path):
"""Does a file or directory exist at the given path?
Like os.path.exists
Parameters
----------
path : string
The API path of a file or directory to check for.
Returns
-------
exists : bool
Whether the target exists.
"""
return self.file_exists(path) or self.dir_exists(path)
def get(self, path, content=True, type=None, format=None):
"""Get a file or directory model."""
raise NotImplementedError('must be implemented in a subclass')
def save(self, model, path):
"""
Save a file or directory model to path.
Should return the saved model with no content. Save implementations
should call self.run_pre_save_hook(model=model, path=path) prior to
writing any data.
"""
raise NotImplementedError('must be implemented in a subclass')
def delete_file(self, path):
"""Delete the file or directory at path."""
raise NotImplementedError('must be implemented in a subclass')
def rename_file(self, old_path, new_path):
"""Rename a file or directory."""
raise NotImplementedError('must be implemented in a subclass')
# ContentsManager API part 2: methods that have useable default
# implementations, but can be overridden in subclasses.
def delete(self, path):
"""Delete a file/directory and any associated checkpoints."""
path = path.strip('/')
if not path:
raise HTTPError(400, "Can't delete root")
self.delete_file(path)
self.checkpoints.delete_all_checkpoints(path)
def rename(self, old_path, new_path):
"""Rename a file and any checkpoints associated with that file."""
self.rename_file(old_path, new_path)
self.checkpoints.rename_all_checkpoints(old_path, new_path)
def update(self, model, path):
"""Update the file's path
For use in PATCH requests, to enable renaming a file without
re-uploading its contents. Only used for renaming at the moment.
"""
path = path.strip('/')
new_path = model.get('path', path).strip('/')
if path != new_path:
self.rename(path, new_path)
model = self.get(new_path, content=False)
return model
def info_string(self):
return "Serving contents"
def get_kernel_path(self, path, model=None):
"""Return the API path for the kernel
KernelManagers can turn this value into a filesystem path,
or ignore it altogether.
The default value here will start kernels in the directory of the
notebook server. FileContentsManager overrides this to use the
directory containing the notebook.
"""
return ''
def increment_filename(self, filename, path='', insert=''):
"""Increment a filename until it is unique.
Parameters
----------
filename : unicode
The name of a file, including extension
path : unicode
The API path of the target's directory
insert: unicode
The characters to insert after the base filename
Returns
-------
name : unicode
A filename that is unique, based on the input filename.
"""
# Extract the full suffix from the filename (e.g. .tar.gz)
path = path.strip('/')
basename, dot, ext = filename.partition('.')
suffix = dot + ext
for i in itertools.count():
if i:
insert_i = '{}{}'.format(insert, i)
else:
insert_i = ''
name = u'{basename}{insert}{suffix}'.format(basename=basename,
insert=insert_i,
suffix=suffix)
if not self.exists(u'{}/{}'.format(path, name)):
break
return name
def validate_notebook_model(self, model):
"""Add failed-validation message to model"""
try:
validate_nb(model['content'])
except ValidationError as e:
model['message'] = u'Notebook validation failed: {}:\n{}'.format(
e.message,
json.dumps(e.instance,
indent=1,
default=lambda obj: '<UNKNOWN>'),
)
return model
def new_untitled(self, path='', type='', ext=''):
"""Create a new untitled file or directory in path
path must be a directory
File extension can be specified.
Use `new` to create files with a fully specified path (including filename).
"""
path = path.strip('/')
if not self.dir_exists(path):
raise HTTPError(404, 'No such directory: %s' % path)
model = {}
if type:
model['type'] = type
if ext == '.ipynb':
model.setdefault('type', 'notebook')
else:
model.setdefault('type', 'file')
insert = ''
if model['type'] == 'directory':
untitled = self.untitled_directory
insert = ' '
elif model['type'] == 'notebook':
untitled = self.untitled_notebook
ext = '.ipynb'
elif model['type'] == 'file':
untitled = self.untitled_file
else:
raise HTTPError(400, "Unexpected model type: %r" % model['type'])
name = self.increment_filename(untitled + ext, path, insert=insert)
path = u'{0}/{1}'.format(path, name)
return self.new(model, path)
def new(self, model=None, path=''):
"""Create a new file or directory and return its model with no content.
To create a new untitled entity in a directory, use `new_untitled`.
"""
path = path.strip('/')
if model is None:
model = {}
if path.endswith('.ipynb'):
model.setdefault('type', 'notebook')
else:
model.setdefault('type', 'file')
# no content, not a directory, so fill out new-file model
if 'content' not in model and model['type'] != 'directory':
if model['type'] == 'notebook':
model['content'] = new_notebook()
model['format'] = 'json'
else:
model['content'] = ''
model['type'] = 'file'
model['format'] = 'text'
model = self.save(model, path)
return model
def copy(self, from_path, to_path=None):
"""Copy an existing file and return its new model.
If to_path not specified, it will be the parent directory of from_path.
If to_path is a directory, filename will increment `from_path-Copy#.ext`.
from_path must be a full path to a file.
"""
path = from_path.strip('/')
if to_path is not None:
to_path = to_path.strip('/')
if '/' in path:
from_dir, from_name = path.rsplit('/', 1)
else:
from_dir = ''
from_name = path
model = self.get(path)
model.pop('path', None)
model.pop('name', None)
if model['type'] == 'directory':
raise HTTPError(400, "Can't copy directories")
if to_path is None:
to_path = from_dir
if self.dir_exists(to_path):
name = copy_pat.sub(u'.', from_name)
to_name = self.increment_filename(name, to_path, insert='-Copy')
to_path = u'{0}/{1}'.format(to_path, to_name)
model = self.save(model, to_path)
return model
def log_info(self):
self.log.info(self.info_string())
def trust_notebook(self, path):
"""Explicitly trust a notebook
Parameters
----------
path : string
The path of a notebook
"""
model = self.get(path)
nb = model['content']
self.log.warning("Trusting notebook %s", path)
self.notary.mark_cells(nb, True)
self.check_and_sign(nb, path)
def check_and_sign(self, nb, path=''):
"""Check for trusted cells, and sign the notebook.
Called as a part of saving notebooks.
Parameters
----------
nb : dict
The notebook dict
path : string
The notebook's path (for logging)
"""
if self.notary.check_cells(nb):
self.notary.sign(nb)
else:
self.log.warning("Notebook %s is not trusted", path)
def mark_trusted_cells(self, nb, path=''):
"""Mark cells as trusted if the notebook signature matches.
Called as a part of loading notebooks.
Parameters
----------
nb : dict
The notebook object (in current nbformat)
path : string
The notebook's path (for logging)
"""
trusted = self.notary.check_signature(nb)
if not trusted:
self.log.warning("Notebook %s is not trusted", path)
self.notary.mark_cells(nb, trusted)
def should_list(self, name):
"""Should this file/directory name be displayed in a listing?"""
return not any(fnmatch(name, glob) for glob in self.hide_globs)
# Part 3: Checkpoints API
def create_checkpoint(self, path):
"""Create a checkpoint."""
return self.checkpoints.create_checkpoint(self, path)
def restore_checkpoint(self, checkpoint_id, path):
"""
Restore a checkpoint.
"""
self.checkpoints.restore_checkpoint(self, checkpoint_id, path)
def list_checkpoints(self, path):
return self.checkpoints.list_checkpoints(path)
def delete_checkpoint(self, checkpoint_id, path):
return self.checkpoints.delete_checkpoint(checkpoint_id, path)
class OAuthenticator(Authenticator):
"""Base class for OAuthenticators
Subclasses must override:
login_service (string identifying the service provider)
authenticate (method takes one arg - the request handler handling the oauth callback)
"""
login_handler = OAuthLoginHandler
callback_handler = OAuthCallbackHandler
logout_handler = OAuthLogoutHandler
authorize_url = Unicode(
config=True, help="""The authenticate url for initiating oauth"""
)
@default("authorize_url")
def _authorize_url_default(self):
return os.environ.get("OAUTH2_AUTHORIZE_URL", "")
token_url = Unicode(
config=True,
help="""The url retrieving an access token at the completion of oauth""",
)
@default("token_url")
def _token_url_default(self):
return os.environ.get("OAUTH2_TOKEN_URL", "")
userdata_url = Unicode(
config=True,
help="""The url for retrieving user data with a completed access token""",
)
@default("userdata_url")
def _userdata_url_default(self):
return os.environ.get("OAUTH2_USERDATA_URL", "")
scope = List(
Unicode(),
config=True,
help="""The OAuth scopes to request.
See the OAuth documentation of your OAuth provider for options.
For GitHub in particular, you can see github_scopes.md in this repo.
""",
)
extra_authorize_params = Dict(
config=True,
help="""Extra GET params to send along with the initial OAuth request
to the OAuth provider.""",
)
login_service = 'override in subclass'
oauth_callback_url = Unicode(
os.getenv('OAUTH_CALLBACK_URL', ''),
config=True,
help="""Callback URL to use.
Typically `https://{host}/hub/oauth_callback`""",
)
client_id_env = ''
client_id = Unicode(config=True)
def _client_id_default(self):
if self.client_id_env:
client_id = os.getenv(self.client_id_env, '')
if client_id:
return client_id
return os.getenv('OAUTH_CLIENT_ID', '')
client_secret_env = ''
client_secret = Unicode(config=True)
def _client_secret_default(self):
if self.client_secret_env:
client_secret = os.getenv(self.client_secret_env, '')
if client_secret:
return client_secret
return os.getenv('OAUTH_CLIENT_SECRET', '')
validate_server_cert_env = 'OAUTH_TLS_VERIFY'
validate_server_cert = Bool(config=True)
def _validate_server_cert_default(self):
env_value = os.getenv(self.validate_server_cert_env, '')
if env_value == '0':
return False
else:
return True
http_client = Any()
@default("http_client")
def _default_http_client(self):
return AsyncHTTPClient()
async def fetch(self, req, label="fetching", parse_json=True, **kwargs):
"""Wrapper for http requests
logs error responses, parses successful JSON responses
Args:
req: tornado HTTPRequest
label (str): label describing what is happening,
used in log message when the request fails.
**kwargs: remaining keyword args
passed to underlying `client.fetch(req, **kwargs)`
Returns:
r: parsed JSON response
"""
try:
resp = await self.http_client.fetch(req, **kwargs)
except HTTPClientError as e:
if e.response:
# Log failed response message for debugging purposes
message = e.response.body.decode("utf8", "replace")
try:
# guess json, reformat for readability
json_message = json.loads(message)
except ValueError:
# not json
pass
else:
# reformat json log message for readability
message = json.dumps(json_message, sort_keys=True, indent=1)
else:
# didn't get a response, e.g. connection error
message = str(e)
# log url without query params
url = urlunparse(urlparse(req.url)._replace(query=""))
app_log.error(f"Error {label} {e.code} {req.method} {url}: {message}")
raise e
else:
if parse_json:
if resp.body:
return json.loads(resp.body.decode('utf8', 'replace'))
else:
# empty body is None
return None
else:
return resp
def login_url(self, base_url):
return url_path_join(base_url, 'oauth_login')
def logout_url(self, base_url):
return url_path_join(base_url, 'logout')
def get_callback_url(self, handler=None):
"""Get my OAuth redirect URL
Either from config or guess based on the current request.
"""
if self.oauth_callback_url:
return self.oauth_callback_url
elif handler:
return guess_callback_uri(
handler.request.protocol,
handler.request.host,
handler.hub.server.base_url,
)
else:
raise ValueError(
"Specify callback oauth_callback_url or give me a handler to guess with"
)
def get_handlers(self, app):
return [
(r'/oauth_login', self.login_handler),
(r'/oauth_callback', self.callback_handler),
(r'/logout', self.logout_handler),
]
async def authenticate(self, handler, data=None):
raise NotImplementedError()
_deprecated_oauth_aliases = {}
def _deprecated_oauth_trait(self, change):
"""observer for deprecated traits"""
old_attr = change.name
new_attr, version = self._deprecated_oauth_aliases.get(old_attr)
new_value = getattr(self, new_attr)
if new_value != change.new:
# only warn if different
# protects backward-compatible config from warnings
# if they set the same value under both names
self.log.warning(
"{cls}.{old} is deprecated in {cls} {version}, use {cls}.{new} instead".format(
cls=self.__class__.__name__,
old=old_attr,
new=new_attr,
version=version,
)
)
setattr(self, new_attr, change.new)
def __init__(self, **kwargs):
# observe deprecated config names in oauthenticator
if self._deprecated_oauth_aliases:
self.observe(
self._deprecated_oauth_trait, names=list(self._deprecated_oauth_aliases)
)
super().__init__(**kwargs)
class TraefikProxy(Proxy):
"""JupyterHub Proxy implementation using traefik"""
traefik_process = Any()
toml_static_config_file = Unicode(
"traefik.toml", config=True, help="""traefik's static configuration file"""
)
traefik_api_url = Unicode(
"http://127.0.0.1:8099",
config=True,
help="""traefik authenticated api endpoint url""",
)
traefik_log_level = Unicode("ERROR", config=True, help="""traefik's log level""")
traefik_api_password = Unicode(
config=True, help="""The password for traefik api login"""
)
@default("traefik_api_password")
def _warn_empty_password(self):
self.log.warning("Traefik API password was not set.")
if self.should_start:
# Generating tokens is fine if the Hub is starting the proxy
self.log.warning("Generating a random token for traefik_api_username...")
return new_token()
self.log.warning(
"Please set c.TraefikProxy.traefik_api_password to authenticate with traefik"
" if the proxy was not started by the Hub."
)
return ""
traefik_api_username = Unicode(
config=True, help="""The username for traefik api login"""
)
@default("traefik_api_username")
def _warn_empty_username(self):
self.log.warning("Traefik API username was not set.")
if self.should_start:
self.log.warning('Defaulting traefik_api_username to "jupyterhub"')
return "jupyterhub"
self.log.warning(
"Please set c.TraefikProxy.traefik_api_username to authenticate with traefik"
" if the proxy was not started by the Hub."
)
return ""
traefik_api_hashed_password = Unicode()
check_route_timeout = Integer(
30,
config=True,
help="""Timeout (in seconds) when waiting for traefik to register an updated route.""",
)
static_config = Dict()
def _generate_htpassword(self):
from passlib.apache import HtpasswdFile
ht = HtpasswdFile()
ht.set_password(self.traefik_api_username, self.traefik_api_password)
self.traefik_api_hashed_password = str(ht.to_string()).split(":")[1][:-3]
async def _check_for_traefik_endpoint(self, routespec, kind, provider):
"""Check for an expected frontend or backend
This is used to wait for traefik to load configuration
from a provider
"""
expected = traefik_utils.generate_alias(routespec, kind)
path = "/api/providers/{}/{}s".format(provider, kind)
try:
resp = await self._traefik_api_request(path)
data = json.loads(resp.body)
except Exception:
self.log.exception("Error checking traefik api for %s %s", kind, routespec)
return False
if expected not in data:
self.log.debug("traefik %s not yet in %ss", expected, kind)
self.log.debug("Current traefik %ss: %s", kind, data)
return False
# found the expected endpoint
return True
async def _wait_for_route(self, routespec, provider):
self.log.info("Waiting for %s to register with traefik", routespec)
async def _check_traefik_dynamic_conf_ready():
"""Check if traefik loaded its dynamic configuration yet"""
if not await self._check_for_traefik_endpoint(
routespec, "backend", provider
):
return False
if not await self._check_for_traefik_endpoint(
routespec, "frontend", provider
):
return False
return True
await exponential_backoff(
_check_traefik_dynamic_conf_ready,
"Traefik route for %s configuration not available" % routespec,
timeout=self.check_route_timeout,
)
async def _traefik_api_request(self, path):
"""Make an API request to traefik"""
url = url_path_join(self.traefik_api_url, path)
self.log.debug("Fetching traefik api %s", url)
resp = await AsyncHTTPClient().fetch(
url,
auth_username=self.traefik_api_username,
auth_password=self.traefik_api_password,
)
if resp.code >= 300:
self.log.warning("%s GET %s", resp.code, url)
else:
self.log.debug("%s GET %s", resp.code, url)
return resp
async def _wait_for_static_config(self, provider):
async def _check_traefik_static_conf_ready():
""" Check if traefik loaded its static configuration from the
etcd cluster """
try:
resp = await self._traefik_api_request("/api/providers/" + provider)
except Exception:
self.log.exception("Error checking for traefik static configuration")
return False
if resp.code != 200:
self.log.error(
"Unexpected response code %s checking for traefik static configuration",
resp.code,
)
return False
return True
await exponential_backoff(
_check_traefik_static_conf_ready,
"Traefik static configuration not available",
timeout=self.check_route_timeout,
)
def _stop_traefik(self):
self.log.info("Cleaning up proxy[%i]...", self.traefik_process.pid)
self.traefik_process.kill()
self.traefik_process.wait()
def _launch_traefik(self, config_type):
if config_type == "toml" or config_type == "etcdv3" or config_type == "consul":
config_file_path = abspath(join(dirname(__file__), "traefik.toml"))
self.traefik_process = Popen(
["traefik", "-c", config_file_path], stdout=None
)
else:
raise ValueError(
"Configuration mode not supported \n.\
The proxy can only be configured through toml and etcd"
)
async def _setup_traefik_static_config(self):
self.log.info("Setting up traefik's static config...")
self._generate_htpassword()
self.static_config = {}
self.static_config["defaultentrypoints"] = ["http"]
self.static_config["debug"] = True
self.static_config["logLevel"] = self.traefik_log_level
entryPoints = {}
entryPoints["http"] = {"address": ":" + str(urlparse(self.public_url).port)}
auth = {
"basic": {
"users": [
self.traefik_api_username + ":" + self.traefik_api_hashed_password
]
}
}
entryPoints["auth_api"] = {
"address": ":" + str(urlparse(self.traefik_api_url).port),
"auth": auth,
}
self.static_config["entryPoints"] = entryPoints
self.static_config["api"] = {"dashboard": True, "entrypoint": "auth_api"}
self.static_config["wss"] = {"protocol": "http"}
async def start(self):
"""Start the proxy.
Will be called during startup if should_start is True.
**Subclasses must define this method**
if the proxy is to be started by the Hub
"""
self._start_traefik()
await self._setup_traefik_static_config()
async def stop(self):
"""Stop the proxy.
Will be called during teardown if should_start is True.
**Subclasses must define this method**
if the proxy is to be started by the Hub
"""
self._stop_traefik()
async def add_route(self, routespec, target, data):
"""Add a route to the proxy.
**Subclasses must define this method**
Args:
routespec (str): A URL prefix ([host]/path/) for which this route will be matched,
e.g. host.name/path/
target (str): A full URL that will be the target of this route.
data (dict): A JSONable dict that will be associated with this route, and will
be returned when retrieving information about this route.
Will raise an appropriate Exception (FIXME: find what?) if the route could
not be added.
The proxy implementation should also have a way to associate the fact that a
route came from JupyterHub.
"""
raise NotImplementedError()
async def delete_route(self, routespec):
"""Delete a route with a given routespec if it exists.
**Subclasses must define this method**
"""
raise NotImplementedError()
async def get_all_routes(self):
"""Fetch and return all the routes associated by JupyterHub from the
proxy.
**Subclasses must define this method**
Should return a dictionary of routes, where the keys are
routespecs and each value is a dict of the form::
{
'routespec': the route specification ([host]/path/)
'target': the target host URL (proto://host) for this route
'data': the attached data dict for this route (as specified in add_route)
}
"""
raise NotImplementedError()
async def get_route(self, routespec):
"""Return the route info for a given routespec.
Args:
routespec (str):
A URI that was used to add this route,
e.g. `host.tld/path/`
Returns:
result (dict):
dict with the following keys::
'routespec': The normalized route specification passed in to add_route
([host]/path/)
'target': The target host for this route (proto://host)
'data': The arbitrary data dict that was passed in by JupyterHub when adding this
route.
None: if there are no routes matching the given routespec
"""
raise NotImplementedError()
class IPythonKernel(KernelBase):
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
shell_class = Type(ZMQInteractiveShell)
use_experimental_completions = Bool(True,
help="Set this flag to False to deactivate the use of experimental IPython completion APIs.",
).tag(config=True)
debugpy_stream = Instance(ZMQStream, allow_none=True)
user_module = Any()
@observe('user_module')
@observe_compat
def _user_module_changed(self, change):
if self.shell is not None:
self.shell.user_module = change['new']
user_ns = Instance(dict, args=None, allow_none=True)
@observe('user_ns')
@observe_compat
def _user_ns_changed(self, change):
if self.shell is not None:
self.shell.user_ns = change['new']
self.shell.init_user_ns()
# A reference to the Python builtin 'raw_input' function.
# (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
_sys_raw_input = Any()
_sys_eval_input = Any()
def __init__(self, **kwargs):
super(IPythonKernel, self).__init__(**kwargs)
# Initialize the Debugger
self.debugger = Debugger(self.log,
self.debugpy_stream,
self._publish_debug_event,
self.debug_shell_socket,
self.session)
# Initialize the InteractiveShell subclass
self.shell = self.shell_class.instance(parent=self,
profile_dir = self.profile_dir,
user_module = self.user_module,
user_ns = self.user_ns,
kernel = self,
compiler_class = XCachingCompiler,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('execute_result')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.comm_manager = CommManager(parent=self, kernel=self)
self.shell.configurables.append(self.comm_manager)
comm_msg_types = [ 'comm_open', 'comm_msg', 'comm_close' ]
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(self.comm_manager, msg_type)
if _use_appnope() and self._darwin_app_nap:
# Disable app-nap as the kernel is not a gui but can have guis
import appnope
appnope.nope()
help_links = List([
{
'text': "Python Reference",
'url': "https://docs.python.org/%i.%i" % sys.version_info[:2],
},
{
'text': "IPython Reference",
'url': "https://ipython.org/documentation.html",
},
{
'text': "NumPy Reference",
'url': "https://docs.scipy.org/doc/numpy/reference/",
},
{
'text': "SciPy Reference",
'url': "https://docs.scipy.org/doc/scipy/reference/",
},
{
'text': "Matplotlib Reference",
'url': "https://matplotlib.org/contents.html",
},
{
'text': "SymPy Reference",
'url': "http://docs.sympy.org/latest/index.html",
},
{
'text': "pandas Reference",
'url': "https://pandas.pydata.org/pandas-docs/stable/",
},
]).tag(config=True)
# Kernel info fields
implementation = 'ipython'
implementation_version = release.version
language_info = {
'name': 'python',
'version': sys.version.split()[0],
'mimetype': 'text/x-python',
'codemirror_mode': {
'name': 'ipython',
'version': sys.version_info[0]
},
'pygments_lexer': 'ipython%d' % 3,
'nbconvert_exporter': 'python',
'file_extension': '.py'
}
@gen.coroutine
def dispatch_debugpy(self, msg):
# The first frame is the socket id, we can drop it
frame = msg[1].bytes.decode('utf-8')
self.log.debug("Debugpy received: %s", frame)
self.debugger.tcp_client.receive_dap_frame(frame)
@property
def banner(self):
return self.shell.banner
def start(self):
self.shell.exit_now = False
self.debugpy_stream.on_recv(self.dispatch_debugpy, copy=False)
super(IPythonKernel, self).start()
def set_parent(self, ident, parent, channel='shell'):
"""Overridden from parent to tell the display hook and output streams
about the parent message.
"""
super(IPythonKernel, self).set_parent(ident, parent, channel)
if channel == 'shell':
self.shell.set_parent(parent)
def init_metadata(self, parent):
"""Initialize metadata.
Run at the beginning of each execution request.
"""
md = super(IPythonKernel, self).init_metadata(parent)
# FIXME: remove deprecated ipyparallel-specific code
# This is required for ipyparallel < 5.0
md.update({
'dependencies_met' : True,
'engine' : self.ident,
})
return md
def finish_metadata(self, parent, metadata, reply_content):
"""Finish populating metadata.
Run after completing an execution request.
"""
# FIXME: remove deprecated ipyparallel-specific code
# This is required by ipyparallel < 5.0
metadata['status'] = reply_content['status']
if reply_content['status'] == 'error' and reply_content['ename'] == 'UnmetDependency':
metadata['dependencies_met'] = False
return metadata
def _forward_input(self, allow_stdin=False):
"""Forward raw_input and getpass to the current frontend.
via input_request
"""
self._allow_stdin = allow_stdin
self._sys_raw_input = builtins.input
builtins.input = self.raw_input
self._save_getpass = getpass.getpass
getpass.getpass = self.getpass
def _restore_input(self):
"""Restore raw_input, getpass"""
builtins.input = self._sys_raw_input
getpass.getpass = self._save_getpass
@property
def execution_count(self):
return self.shell.execution_count
@execution_count.setter
def execution_count(self, value):
# Ignore the incrementing done by KernelBase, in favour of our shell's
# execution counter.
pass
@contextmanager
def _cancel_on_sigint(self, future):
"""ContextManager for capturing SIGINT and cancelling a future
SIGINT raises in the event loop when running async code,
but we want it to halt a coroutine.
Ideally, it would raise KeyboardInterrupt,
but this turns it into a CancelledError.
At least it gets a decent traceback to the user.
"""
sigint_future = asyncio.Future()
# whichever future finishes first,
# cancel the other one
def cancel_unless_done(f, _ignored):
if f.cancelled() or f.done():
return
f.cancel()
# when sigint finishes,
# abort the coroutine with CancelledError
sigint_future.add_done_callback(
partial(cancel_unless_done, future)
)
# when the main future finishes,
# stop watching for SIGINT events
future.add_done_callback(
partial(cancel_unless_done, sigint_future)
)
def handle_sigint(*args):
def set_sigint_result():
if sigint_future.cancelled() or sigint_future.done():
return
sigint_future.set_result(1)
# use add_callback for thread safety
self.io_loop.add_callback(set_sigint_result)
# set the custom sigint hander during this context
save_sigint = signal.signal(signal.SIGINT, handle_sigint)
try:
yield
finally:
# restore the previous sigint handler
signal.signal(signal.SIGINT, save_sigint)
@gen.coroutine
def do_execute(self, code, silent, store_history=True,
user_expressions=None, allow_stdin=False):
shell = self.shell # we'll need this a lot here
self._forward_input(allow_stdin)
reply_content = {}
if hasattr(shell, 'run_cell_async') and hasattr(shell, 'should_run_async'):
run_cell = shell.run_cell_async
should_run_async = shell.should_run_async
else:
should_run_async = lambda cell: False
# older IPython,
# use blocking run_cell and wrap it in coroutine
@gen.coroutine
def run_cell(*args, **kwargs):
return shell.run_cell(*args, **kwargs)
try:
# default case: runner is asyncio and asyncio is already running
# TODO: this should check every case for "are we inside the runner",
# not just asyncio
preprocessing_exc_tuple = None
try:
transformed_cell = self.shell.transform_cell(code)
except Exception:
transformed_cell = code
preprocessing_exc_tuple = sys.exc_info()
if (
_asyncio_runner
and shell.loop_runner is _asyncio_runner
and asyncio.get_event_loop().is_running()
and should_run_async(code, transformed_cell=transformed_cell, preprocessing_exc_tuple=preprocessing_exc_tuple)
):
coro = run_cell(
code,
store_history=store_history,
silent=silent,
transformed_cell=transformed_cell,
preprocessing_exc_tuple=preprocessing_exc_tuple
)
coro_future = asyncio.ensure_future(coro)
with self._cancel_on_sigint(coro_future):
res = None
try:
res = yield coro_future
finally:
shell.events.trigger('post_execute')
if not silent:
shell.events.trigger('post_run_cell', res)
else:
# runner isn't already running,
# make synchronous call,
# letting shell dispatch to loop runners
res = shell.run_cell(code, store_history=store_history, silent=silent)
finally:
self._restore_input()
if res.error_before_exec is not None:
err = res.error_before_exec
else:
err = res.error_in_exec
if res.success:
reply_content['status'] = 'ok'
else:
reply_content['status'] = 'error'
reply_content.update({
'traceback': shell._last_traceback or [],
'ename': str(type(err).__name__),
'evalue': safe_unicode(err),
})
# FIXME: deprecated piece for ipyparallel (remove in 5.0):
e_info = dict(engine_uuid=self.ident, engine_id=self.int_id,
method='execute')
reply_content['engine_info'] = e_info
# Return the execution counter so clients can display prompts
reply_content['execution_count'] = shell.execution_count - 1
if 'traceback' in reply_content:
self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback']))
# At this point, we can tell whether the main code execution succeeded
# or not. If it did, we proceed to evaluate user_expressions
if reply_content['status'] == 'ok':
reply_content['user_expressions'] = \
shell.user_expressions(user_expressions or {})
else:
# If there was an error, don't even try to compute expressions
reply_content['user_expressions'] = {}
# Payloads should be retrieved regardless of outcome, so we can both
# recover partial output (that could have been generated early in a
# block, before an error) and always clear the payload system.
reply_content['payload'] = shell.payload_manager.read_payload()
# Be aggressive about clearing the payload because we don't want
# it to sit in memory until the next execute_request comes in.
shell.payload_manager.clear_payload()
return reply_content
def do_complete(self, code, cursor_pos):
if _use_experimental_60_completion and self.use_experimental_completions:
return self._experimental_do_complete(code, cursor_pos)
# FIXME: IPython completers currently assume single line,
# but completion messages give multi-line context
# For now, extract line from cell, based on cursor_pos:
if cursor_pos is None:
cursor_pos = len(code)
line, offset = line_at_cursor(code, cursor_pos)
line_cursor = cursor_pos - offset
txt, matches = self.shell.complete('', line, line_cursor)
return {'matches' : matches,
'cursor_end' : cursor_pos,
'cursor_start' : cursor_pos - len(txt),
'metadata' : {},
'status' : 'ok'}
@gen.coroutine
def do_debug_request(self, msg):
return (yield self.debugger.process_request(msg))
def _experimental_do_complete(self, code, cursor_pos):
"""
Experimental completions from IPython, using Jedi.
"""
if cursor_pos is None:
cursor_pos = len(code)
with _provisionalcompleter():
raw_completions = self.shell.Completer.completions(code, cursor_pos)
completions = list(_rectify_completions(code, raw_completions))
comps = []
for comp in completions:
comps.append(dict(
start=comp.start,
end=comp.end,
text=comp.text,
type=comp.type,
))
if completions:
s = completions[0].start
e = completions[0].end
matches = [c.text for c in completions]
else:
s = cursor_pos
e = cursor_pos
matches = []
return {'matches': matches,
'cursor_end': e,
'cursor_start': s,
'metadata': {_EXPERIMENTAL_KEY_NAME: comps},
'status': 'ok'}
def do_inspect(self, code, cursor_pos, detail_level=0):
name = token_at_cursor(code, cursor_pos)
reply_content = {'status' : 'ok'}
reply_content['data'] = {}
reply_content['metadata'] = {}
try:
reply_content['data'].update(
self.shell.object_inspect_mime(
name,
detail_level=detail_level
)
)
if not self.shell.enable_html_pager:
reply_content['data'].pop('text/html')
reply_content['found'] = True
except KeyError:
reply_content['found'] = False
return reply_content
def do_history(self, hist_access_type, output, raw, session=0, start=0,
stop=None, n=None, pattern=None, unique=False):
if hist_access_type == 'tail':
hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
include_latest=True)
elif hist_access_type == 'range':
hist = self.shell.history_manager.get_range(session, start, stop,
raw=raw, output=output)
elif hist_access_type == 'search':
hist = self.shell.history_manager.search(
pattern, raw=raw, output=output, n=n, unique=unique)
else:
hist = []
return {
'status': 'ok',
'history' : list(hist),
}
def do_shutdown(self, restart):
self.shell.exit_now = True
return dict(status='ok', restart=restart)
def do_is_complete(self, code):
transformer_manager = getattr(self.shell, 'input_transformer_manager', None)
if transformer_manager is None:
# input_splitter attribute is deprecated
transformer_manager = self.shell.input_splitter
status, indent_spaces = transformer_manager.check_complete(code)
r = {'status': status}
if status == 'incomplete':
r['indent'] = ' ' * indent_spaces
return r
def do_apply(self, content, bufs, msg_id, reply_metadata):
from .serialize import serialize_object, unpack_apply_message
shell = self.shell
try:
working = shell.user_ns
prefix = "_"+str(msg_id).replace("-","")+"_"
f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
fname = getattr(f, '__name__', 'f')
fname = prefix+"f"
argname = prefix+"args"
kwargname = prefix+"kwargs"
resultname = prefix+"result"
ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
# print ns
working.update(ns)
code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname)
try:
exec(code, shell.user_global_ns, shell.user_ns)
result = working.get(resultname)
finally:
for key in ns:
working.pop(key)
result_buf = serialize_object(result,
buffer_threshold=self.session.buffer_threshold,
item_threshold=self.session.item_threshold,
)
except BaseException as e:
# invoke IPython traceback formatting
shell.showtraceback()
reply_content = {
'traceback': shell._last_traceback or [],
'ename': str(type(e).__name__),
'evalue': safe_unicode(e),
}
# FIXME: deprecated piece for ipyparallel (remove in 5.0):
e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply')
reply_content['engine_info'] = e_info
self.send_response(self.iopub_socket, 'error', reply_content,
ident=self._topic('error'), channel='shell')
self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback']))
result_buf = []
reply_content['status'] = 'error'
else:
reply_content = {'status' : 'ok'}
return reply_content, result_buf
def do_clear(self):
self.shell.reset(False)
return dict(status='ok')
class TerminalInteractiveShell(InteractiveShell):
space_for_menu = Integer(
6,
help='Number of line at the bottom of the screen '
'to reserve for the completion menu').tag(config=True)
def _space_for_menu_changed(self, old, new):
self._update_layout()
pt_cli = None
debugger_history = None
_pt_app = None
simple_prompt = Bool(
_use_simple_prompt,
help=
"""Use `raw_input` for the REPL, without completion, multiline input, and prompt colors.
Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
IPython own testing machinery, and emacs inferior-shell integration through elpy.
This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
environment variable is set, or the current terminal is not a tty."""
).tag(config=True)
@property
def debugger_cls(self):
return Pdb if self.simple_prompt else TerminalPdb
confirm_exit = Bool(
True,
help="""
Set to confirm when you try to exit IPython with an EOF (Control-D
in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
you can force a direct exit without any confirmation.""",
).tag(config=True)
editing_mode = Unicode(
'emacs',
help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
).tag(config=True)
mouse_support = Bool(
False, help="Enable mouse support in the prompt").tag(config=True)
highlighting_style = Union(
[Unicode('legacy'), Type(klass=Style)],
help="""The name or class of a Pygments style to use for syntax
highlighting: \n %s""" % ', '.join(get_all_styles())).tag(config=True)
@observe('highlighting_style')
@observe('colors')
def _highlighting_style_changed(self, change):
self.refresh_style()
def refresh_style(self):
self._style = self._make_style_from_name_or_cls(
self.highlighting_style)
highlighting_style_overrides = Dict(
help="Override highlighting format for specific tokens").tag(
config=True)
true_color = Bool(
False,
help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
"If your terminal supports true color, the following command "
"should print 'TRUECOLOR' in orange: "
"printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")).tag(
config=True)
editor = Unicode(
get_default_editor(),
help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
).tag(config=True)
prompts_class = Type(
Prompts,
help='Class used to generate Prompt token for prompt_toolkit').tag(
config=True)
prompts = Instance(Prompts)
@default('prompts')
def _prompts_default(self):
return self.prompts_class(self)
@observe('prompts')
def _(self, change):
self._update_layout()
@default('displayhook_class')
def _displayhook_class_default(self):
return RichPromptDisplayHook
term_title = Bool(
True, help="Automatically set the terminal title").tag(config=True)
display_completions = Enum(
('column', 'multicolumn', 'readlinelike'),
help=
("Options for displaying tab completions, 'column', 'multicolumn', and "
"'readlinelike'. These options are for `prompt_toolkit`, see "
"`prompt_toolkit` documentation for more information."),
default_value='multicolumn').tag(config=True)
highlight_matching_brackets = Bool(
True,
help="Highlight matching brackets.",
).tag(config=True)
extra_open_editor_shortcuts = Bool(
False,
help=
"Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
"This is in addition to the F2 binding, which is always enabled.").tag(
config=True)
handle_return = Any(
None,
help="Provide an alternative handler to be called when the user presses "
"Return. This is an advanced option intended for debugging, which "
"may be changed or removed in later releases.").tag(config=True)
@observe('term_title')
def init_term_title(self, change=None):
# Enable or disable the terminal title.
if self.term_title:
toggle_set_term_title(True)
set_term_title('IPython: ' + abbrev_cwd())
else:
toggle_set_term_title(False)
def init_display_formatter(self):
super(TerminalInteractiveShell, self).init_display_formatter()
# terminal only supports plain text
self.display_formatter.active_types = ['text/plain']
# disable `_ipython_display_`
self.display_formatter.ipython_display_formatter.enabled = False
def init_prompt_toolkit_cli(self):
if self.simple_prompt:
# Fall back to plain non-interactive output for tests.
# This is very limited, and only accepts a single line.
def prompt():
return cast_unicode_py2(
input('In [%d]: ' % self.execution_count))
self.prompt_for_code = prompt
return
# Set up keyboard shortcuts
kbmanager = KeyBindingManager.for_prompt(
enable_open_in_editor=self.extra_open_editor_shortcuts, )
register_ipython_shortcuts(kbmanager.registry, self)
# Pre-populate history from IPython's history database
history = InMemoryHistory()
last_cell = u""
for __, ___, cell in self.history_manager.get_tail(
self.history_load_length, include_latest=True):
# Ignore blank lines and consecutive duplicates
cell = cell.rstrip()
if cell and (cell != last_cell):
history.append(cell)
last_cell = cell
self._style = self._make_style_from_name_or_cls(
self.highlighting_style)
self.style = DynamicStyle(lambda: self._style)
editing_mode = getattr(EditingMode, self.editing_mode.upper())
def patch_stdout(**kwargs):
return self.pt_cli.patch_stdout_context(**kwargs)
self._pt_app = create_prompt_application(
editing_mode=editing_mode,
key_bindings_registry=kbmanager.registry,
history=history,
completer=IPythonPTCompleter(shell=self,
patch_stdout=patch_stdout),
enable_history_search=True,
style=self.style,
mouse_support=self.mouse_support,
**self._layout_options())
self._eventloop = create_eventloop(self.inputhook)
self.pt_cli = CommandLineInterface(
self._pt_app,
eventloop=self._eventloop,
output=create_output(true_color=self.true_color))
def _make_style_from_name_or_cls(self, name_or_cls):
"""
Small wrapper that make an IPython compatible style from a style name
We need that to add style for prompt ... etc.
"""
style_overrides = {}
if name_or_cls == 'legacy':
legacy = self.colors.lower()
if legacy == 'linux':
style_cls = get_style_by_name('monokai')
style_overrides = _style_overrides_linux
elif legacy == 'lightbg':
style_overrides = _style_overrides_light_bg
style_cls = get_style_by_name('pastie')
elif legacy == 'neutral':
# The default theme needs to be visible on both a dark background
# and a light background, because we can't tell what the terminal
# looks like. These tweaks to the default theme help with that.
style_cls = get_style_by_name('default')
style_overrides.update({
Token.Number: '#007700',
Token.Operator: 'noinherit',
Token.String: '#BB6622',
Token.Name.Function: '#2080D0',
Token.Name.Class: 'bold #2080D0',
Token.Name.Namespace: 'bold #2080D0',
Token.Prompt: '#009900',
Token.PromptNum: '#00ff00 bold',
Token.OutPrompt: '#990000',
Token.OutPromptNum: '#ff0000 bold',
})
# Hack: Due to limited color support on the Windows console
# the prompt colors will be wrong without this
if os.name == 'nt':
style_overrides.update({
Token.Prompt: '#ansidarkgreen',
Token.PromptNum: '#ansigreen bold',
Token.OutPrompt: '#ansidarkred',
Token.OutPromptNum: '#ansired bold',
})
elif legacy == 'nocolor':
style_cls = _NoStyle
style_overrides = {}
else:
raise ValueError('Got unknown colors: ', legacy)
else:
if isinstance(name_or_cls, str):
style_cls = get_style_by_name(name_or_cls)
else:
style_cls = name_or_cls
style_overrides = {
Token.Prompt: '#009900',
Token.PromptNum: '#00ff00 bold',
Token.OutPrompt: '#990000',
Token.OutPromptNum: '#ff0000 bold',
}
style_overrides.update(self.highlighting_style_overrides)
style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls,
style_dict=style_overrides)
return style
def _layout_options(self):
"""
Return the current layout option for the current Terminal InteractiveShell
"""
return {
'lexer':
IPythonPTLexer(),
'reserve_space_for_menu':
self.space_for_menu,
'get_prompt_tokens':
self.prompts.in_prompt_tokens,
'get_continuation_tokens':
self.prompts.continuation_prompt_tokens,
'multiline':
True,
'display_completions_in_columns':
(self.display_completions == 'multicolumn'),
# Highlight matching brackets, but only when this setting is
# enabled, and only when the DEFAULT_BUFFER has the focus.
'extra_input_processors': [
ConditionalProcessor(
processor=HighlightMatchingBracketProcessor(
chars='[](){}'),
filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()
& Condition(lambda cli: self.highlight_matching_brackets))
],
}
def _update_layout(self):
"""
Ask for a re computation of the application layout, if for example ,
some configuration options have changed.
"""
if self._pt_app:
self._pt_app.layout = create_prompt_layout(
**self._layout_options())
def prompt_for_code(self):
document = self.pt_cli.run(pre_run=self.pre_prompt,
reset_current_buffer=True)
return document.text
def enable_win_unicode_console(self):
if sys.version_info >= (3, 6):
# Since PEP 528, Python uses the unicode APIs for the Windows
# console by default, so WUC shouldn't be needed.
return
import win_unicode_console
win_unicode_console.enable()
def init_io(self):
if sys.platform not in {'win32', 'cli'}:
return
self.enable_win_unicode_console()
import colorama
colorama.init()
# For some reason we make these wrappers around stdout/stderr.
# For now, we need to reset them so all output gets coloured.
# https://github.com/ipython/ipython/issues/8669
# io.std* are deprecated, but don't show our own deprecation warnings
# during initialization of the deprecated API.
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
io.stdout = io.IOStream(sys.stdout)
io.stderr = io.IOStream(sys.stderr)
def init_magics(self):
super(TerminalInteractiveShell, self).init_magics()
self.register_magics(TerminalMagics)
def init_alias(self):
# The parent class defines aliases that can be safely used with any
# frontend.
super(TerminalInteractiveShell, self).init_alias()
# Now define aliases that only make sense on the terminal, because they
# need direct access to the console in a way that we can't emulate in
# GUI or web frontend
if os.name == 'posix':
for cmd in ['clear', 'more', 'less', 'man']:
self.alias_manager.soft_define_alias(cmd, cmd)
def __init__(self, *args, **kwargs):
super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
self.init_prompt_toolkit_cli()
self.init_term_title()
self.keep_running = True
self.debugger_history = InMemoryHistory()
def ask_exit(self):
self.keep_running = False
rl_next_input = None
def pre_prompt(self):
if self.rl_next_input:
# We can't set the buffer here, because it will be reset just after
# this. Adding a callable to pre_run_callables does what we need
# after the buffer is reset.
s = cast_unicode_py2(self.rl_next_input)
def set_doc():
self.pt_cli.application.buffer.document = Document(s)
if hasattr(self.pt_cli, 'pre_run_callables'):
self.pt_cli.pre_run_callables.append(set_doc)
else:
# Older version of prompt_toolkit; it's OK to set the document
# directly here.
set_doc()
self.rl_next_input = None
def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
if display_banner is not DISPLAY_BANNER_DEPRECATED:
warn(
'interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.',
DeprecationWarning,
stacklevel=2)
self.keep_running = True
while self.keep_running:
print(self.separate_in, end='')
try:
code = self.prompt_for_code()
except EOFError:
if (not self.confirm_exit) \
or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
self.ask_exit()
else:
if code:
self.run_cell(code, store_history=True)
def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
# An extra layer of protection in case someone mashing Ctrl-C breaks
# out of our internal code.
if display_banner is not DISPLAY_BANNER_DEPRECATED:
warn(
'mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.',
DeprecationWarning,
stacklevel=2)
while True:
try:
self.interact()
break
except KeyboardInterrupt as e:
print("\n%s escaped interact()\n" % type(e).__name__)
finally:
# An interrupt during the eventloop will mess up the
# internal state of the prompt_toolkit library.
# Stopping the eventloop fixes this, see
# https://github.com/ipython/ipython/pull/9867
if hasattr(self, '_eventloop'):
self._eventloop.stop()
_inputhook = None
def inputhook(self, context):
if self._inputhook is not None:
self._inputhook(context)
active_eventloop = None
def enable_gui(self, gui=None):
if gui:
self.active_eventloop, self._inputhook =\
get_inputhook_name_and_func(gui)
else:
self.active_eventloop = self._inputhook = None
# Run !system commands directly, not through pipes, so terminal programs
# work correctly.
system = InteractiveShell.system_raw
def auto_rewrite_input(self, cmd):
"""Overridden from the parent class to use fancy rewriting prompt"""
if not self.show_rewritten_input:
return
tokens = self.prompts.rewrite_prompt_tokens()
if self.pt_cli:
self.pt_cli.print_tokens(tokens)
print(cmd)
else:
prompt = ''.join(s for t, s in tokens)
print(prompt, cmd, sep='')
_prompts_before = None
def switch_doctest_mode(self, mode):
"""Switch prompts to classic for %doctest_mode"""
if mode:
self._prompts_before = self.prompts
self.prompts = ClassicPrompts(self)
elif self._prompts_before:
self.prompts = self._prompts_before
self._prompts_before = None
self._update_layout()
class Service(LoggingConfigurable):
"""An object wrapping a service specification for Hub API consumers.
A service has inputs:
- name: str
the name of the service
- admin: bool(false)
whether the service should have administrative privileges
- url: str (None)
The URL where the service is/should be.
If specified, the service will be added to the proxy at /services/:name
If a service is to be managed by the Hub, it has a few extra options:
- command: (str/Popen list)
Command for JupyterHub to spawn the service.
Only use this if the service should be a subprocess.
If command is not specified, it is assumed to be managed
by a
- environment: dict
Additional environment variables for the service.
- user: str
The name of a system user to become.
If unspecified, run as the same user as the Hub.
"""
# inputs:
name = Unicode(
help="""The name of the service.
If the service has an http endpoint, it
"""
).tag(input=True)
admin = Bool(False,
help="Does the service need admin-access to the Hub API?"
).tag(input=True)
url = Unicode(
help="""URL of the service.
Only specify if the service runs an HTTP(s) endpoint that.
If managed, will be passed as JUPYTERHUB_SERVICE_URL env.
"""
).tag(input=True)
api_token = Unicode(
help="""The API token to use for the service.
If unspecified, an API token will be generated for managed services.
"""
).tag(input=True)
# Managed service API:
@property
def managed(self):
"""Am I managed by the Hub?"""
return bool(self.command)
@property
def kind(self):
"""The name of the kind of service as a string
- 'managed' for managed services
- 'external' for external services
"""
return 'managed' if self.managed else 'external'
command = Command(minlen=0,
help="Command to spawn this service, if managed."
).tag(input=True)
cwd = Unicode(
help="""The working directory in which to run the service."""
).tag(input=True)
environment = Dict(
help="""Environment variables to pass to the service.
Only used if the Hub is spawning the service.
"""
).tag(input=True)
user = Unicode(getuser(),
help="""The user to become when launching the service.
If unspecified, run the service as the same user as the Hub.
"""
).tag(input=True)
domain = Unicode()
host = Unicode()
hub = Any()
proc = Any()
# handles on globals:
proxy = Any()
base_url = Unicode()
db = Any()
orm = Any()
oauth_provider = Any()
oauth_client_id = Unicode(
help="""OAuth client ID for this service.
You shouldn't generally need to change this.
Default: `service-<name>`
"""
).tag(input=True)
@default('oauth_client_id')
def _default_client_id(self):
return 'service-%s' % self.name
@property
def server(self):
return self.orm.server
@property
def prefix(self):
return url_path_join(self.base_url, 'services', self.name + '/')
@property
def proxy_path(self):
if not self.server:
return ''
if self.domain:
return url_path_join('/' + self.domain, self.server.base_url)
else:
return self.server.base_url
def __repr__(self):
return "<{cls}(name={name}{managed})>".format(
cls=self.__class__.__name__,
name=self.name,
managed=' managed' if self.managed else '',
)
def start(self):
"""Start a managed service"""
if not self.managed:
raise RuntimeError("Cannot start unmanaged service %s" % self)
self.log.info("Starting service %r: %r", self.name, self.command)
env = {}
env.update(self.environment)
env['JUPYTERHUB_SERVICE_NAME'] = self.name
env['JUPYTERHUB_API_TOKEN'] = self.api_token
env['JUPYTERHUB_API_URL'] = self.hub_api_url
env['JUPYTERHUB_BASE_URL'] = self.base_url
if self.url:
env['JUPYTERHUB_SERVICE_URL'] = self.url
env['JUPYTERHUB_SERVICE_PREFIX'] = self.server.base_url
self.spawner = _ServiceSpawner(
cmd=self.command,
environment=env,
api_token=self.api_token,
oauth_client_id=self.oauth_client_id,
cwd=self.cwd,
hub=self.hub,
user=_MockUser(
name=self.user,
service=self,
server=self.orm.server,
host=self.host,
),
)
self.spawner.start()
self.proc = self.spawner.proc
self.spawner.add_poll_callback(self._proc_stopped)
self.spawner.start_polling()
def _proc_stopped(self):
"""Called when the service process unexpectedly exits"""
self.log.error("Service %s exited with status %i", self.name, self.proc.returncode)
self.start()
def stop(self):
"""Stop a managed service"""
if not self.managed:
raise RuntimeError("Cannot start unmanaged service %s" % self)
self.spawner.stop_polling()
return self.spawner.stop()
class PapayaWidget(HBox):
"""A widget class that displays a papaya viewer (NlPapayaViewer) and config widget (PapayaConfigWidget) side by side."""
current_config = Any()
current_colorbar = Any()
def __init__(self, *args, **kwargs):
"""
Parameters
----------
**kwargs:
config_visible: str
Depending on the value, config widget will be visible or hidden upon initialization. Possible values are "hidden" or "visible"
"""
super().__init__(*args, **kwargs)
self._viewer = NlPapayaViewer(layout=Layout(
width="70%", height="auto", border="1px solid black"))
self._config = PapayaConfigWidget(
self._viewer,
layout=Layout(width="30%", height="auto",
border="1px solid black"),
)
self._config.layout.visibility = kwargs.get("config_visible", "hidden")
self.current_config = None
self.current_colorbar = None
self.children = [self._viewer, self._config]
def show_image_config(self, image, show=True):
if show:
self._config.layout.visibility = "visible"
self.current_config = image
self._config.set_image(image)
else:
if self.current_config is not None and image.id == self.current_config.id:
self._config.layout.visibility = "hidden"
self._config.set_image(None)
def show_image_colorbar(self, image):
self.current_colorbar = image
self._viewer.show_image_colorbar(image)
def can_add(self, images):
return self._viewer.can_add(images)
def add(self, images):
self._viewer.add(images)
self.current_colorbar = self._viewer.get_colorbar_image()
def remove(self, images):
self._viewer.remove(images)
self.current_colorbar = self._viewer.get_colorbar_image()
def set_images(self):
self._viewer.set_images()
def set_center(self, widget, image):
self._viewer.set_center(widget, image)
def reset_center(self):
self._viewer.reset_center()
def set_error(self, error):
self._viewer.set_error(error)
def reset(self):
self._viewer.reset()
self._config.reset()
def get_hex_for_lut(self, lut):
return self._viewer.get_hex_for_lut(lut)
class Authenticator(LoggingConfigurable):
"""Base class for implementing an authentication provider for JupyterHub"""
db = Any()
admin_users = Set(
help="""
Set of users that will have admin rights on this JupyterHub.
Admin users have extra privileges:
- Use the admin panel to see list of users logged in
- Add / remove users in some authenticators
- Restart / halt the hub
- Start / stop users' single-user servers
- Can access each individual users' single-user server (if configured)
Admin access should be treated the same way root access is.
Defaults to an empty set, in which case no user has admin access.
"""
).tag(config=True)
whitelist = Set(
help="""
Whitelist of usernames that are allowed to log in.
Use this with supported authenticators to restrict which users can log in. This is an
additional whitelist that further restricts users, beyond whatever restrictions the
authenticator has in place.
If empty, does not perform any additional restriction.
"""
).tag(config=True)
@observe('whitelist')
def _check_whitelist(self, change):
short_names = [name for name in change['new'] if len(name) <= 1]
if short_names:
sorted_names = sorted(short_names)
single = ''.join(sorted_names)
string_set_typo = "set('%s')" % single
self.log.warning("whitelist contains single-character names: %s; did you mean set([%r]) instead of %s?",
sorted_names[:8], single, string_set_typo,
)
custom_html = Unicode(
help="""
HTML form to be overridden by authenticators if they want a custom authentication form.
Defaults to an empty string, which shows the default username/password form.
"""
)
login_service = Unicode(
help="""
Name of the login service that this authenticator is providing using to authenticate users.
Example: GitHub, MediaWiki, Google, etc.
Setting this value replaces the login form with a "Login with <login_service>" button.
Any authenticator that redirects to an external service (e.g. using OAuth) should set this.
"""
)
username_pattern = Unicode(
help="""
Regular expression pattern that all valid usernames must match.
If a username does not match the pattern specified here, authentication will not be attempted.
If not set, allow any username.
"""
).tag(config=True)
@observe('username_pattern')
def _username_pattern_changed(self, change):
if not change['new']:
self.username_regex = None
self.username_regex = re.compile(change['new'])
username_regex = Any(
help="""
Compiled regex kept in sync with `username_pattern`
"""
)
def validate_username(self, username):
"""Validate a normalized username
Return True if username is valid, False otherwise.
"""
if not self.username_regex:
return True
return bool(self.username_regex.match(username))
username_map = Dict(
help="""Dictionary mapping authenticator usernames to JupyterHub users.
Primarily used to normalize OAuth user names to local users.
"""
).tag(config=True)
delete_invalid_users = Bool(False,
help="""Delete any users from the database that do not pass validation
When JupyterHub starts, `.add_user` will be called
on each user in the database to verify that all users are still valid.
If `delete_invalid_users` is True,
any users that do not pass validation will be deleted from the database.
Use this if users might be deleted from an external system,
such as local user accounts.
If False (default), invalid users remain in the Hub's database
and a warning will be issued.
This is the default to avoid data loss due to config changes.
"""
)
def normalize_username(self, username):
"""Normalize the given username and return it
Override in subclasses if usernames need different normalization rules.
The default attempts to lowercase the username and apply `username_map` if it is
set.
"""
username = username.lower()
username = self.username_map.get(username, username)
return username
def check_whitelist(self, username):
"""Check if a username is allowed to authenticate based on whitelist configuration
Return True if username is allowed, False otherwise.
No whitelist means any username is allowed.
Names are normalized *before* being checked against the whitelist.
"""
if not self.whitelist:
# No whitelist means any name is allowed
return True
return username in self.whitelist
@gen.coroutine
def get_authenticated_user(self, handler, data):
"""Authenticate the user who is attempting to log in
Returns normalized username if successful, None otherwise.
This calls `authenticate`, which should be overridden in subclasses,
normalizes the username if any normalization should be done,
and then validates the name in the whitelist.
This is the outer API for authenticating a user.
Subclasses should not need to override this method.
The various stages can be overridden separately:
- `authenticate` turns formdata into a username
- `normalize_username` normalizes the username
- `check_whitelist` checks against the user whitelist
"""
username = yield self.authenticate(handler, data)
if username is None:
return
username = self.normalize_username(username)
if not self.validate_username(username):
self.log.warning("Disallowing invalid username %r.", username)
return
whitelist_pass = yield gen.maybe_future(self.check_whitelist(username))
if whitelist_pass:
return username
else:
self.log.warning("User %r not in whitelist.", username)
return
@gen.coroutine
def authenticate(self, handler, data):
"""Authenticate a user with login form data
This must be a tornado gen.coroutine.
It must return the username on successful authentication,
and return None on failed authentication.
Checking the whitelist is handled separately by the caller.
Args:
handler (tornado.web.RequestHandler): the current request handler
data (dict): The formdata of the login form.
The default form has 'username' and 'password' fields.
Returns:
username (str or None): The username of the authenticated user,
or None if Authentication failed
"""
def pre_spawn_start(self, user, spawner):
"""Hook called before spawning a user's server
Can be used to do auth-related startup, e.g. opening PAM sessions.
"""
def post_spawn_stop(self, user, spawner):
"""Hook called after stopping a user container
Can be used to do auth-related cleanup, e.g. closing PAM sessions.
"""
def add_user(self, user):
"""Hook called when a user is added to JupyterHub
This is called:
- When a user first authenticates
- When the hub restarts, for all users.
This method may be a coroutine.
By default, this just adds the user to the whitelist.
Subclasses may do more extensive things, such as adding actual unix users,
but they should call super to ensure the whitelist is updated.
Note that this should be idempotent, since it is called whenever the hub restarts
for all users.
Args:
user (User): The User wrapper object
"""
if not self.validate_username(user.name):
raise ValueError("Invalid username: %s" % user.name)
if self.whitelist:
self.whitelist.add(user.name)
def delete_user(self, user):
"""Hook called when a user is deleted
Removes the user from the whitelist.
Subclasses should call super to ensure the whitelist is updated.
Args:
user (User): The User wrapper object
"""
self.whitelist.discard(user.name)
auto_login = Bool(False, config=True,
help="""Automatically begin the login process
rather than starting with a "Login with..." link at `/hub/login`
To work, `.login_url()` must give a URL other than the default `/hub/login`,
such as an oauth handler or another automatic login handler,
registered with `.get_handlers()`.
.. versionadded:: 0.8
"""
)
def login_url(self, base_url):
"""Override this when registering a custom login handler
Generally used by authenticators that do not use simple form-based authentication.
The subclass overriding this is responsible for making sure there is a handler
available to handle the URL returned from this method, using the `get_handlers`
method.
Args:
base_url (str): the base URL of the Hub (e.g. /hub/)
Returns:
str: The login URL, e.g. '/hub/login'
"""
return url_path_join(base_url, 'login')
def logout_url(self, base_url):
"""Override when registering a custom logout handler
The subclass overriding this is responsible for making sure there is a handler
available to handle the URL returned from this method, using the `get_handlers`
method.
Args:
base_url (str): the base URL of the Hub (e.g. /hub/)
Returns:
str: The logout URL, e.g. '/hub/logout'
"""
return url_path_join(base_url, 'logout')
def get_handlers(self, app):
"""Return any custom handlers the authenticator needs to register
Used in conjugation with `login_url` and `logout_url`.
Args:
app (JupyterHub Application):
the application object, in case it needs to be accessed for info.
Returns:
handlers (list):
list of ``('/url', Handler)`` tuples passed to tornado.
The Hub prefix is added to any URLs.
"""
return [
('/login', LoginHandler),
]
class TerminalInteractiveShell(InteractiveShell):
mime_renderers = Dict().tag(config=True)
space_for_menu = Integer(
6,
help='Number of line at the bottom of the screen '
'to reserve for the tab completion menu, '
'search history, ...etc, the height of '
'these menus will at most this value. '
'Increase it is you prefer long and skinny '
'menus, decrease for short and wide.').tag(config=True)
pt_app = None
debugger_history = None
simple_prompt = Bool(
_use_simple_prompt,
help=
"""Use `raw_input` for the REPL, without completion and prompt colors.
Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
IPython own testing machinery, and emacs inferior-shell integration through elpy.
This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
environment variable is set, or the current terminal is not a tty."""
).tag(config=True)
@property
def debugger_cls(self):
return Pdb if self.simple_prompt else TerminalPdb
confirm_exit = Bool(
True,
help="""
Set to confirm when you try to exit IPython with an EOF (Control-D
in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
you can force a direct exit without any confirmation.""",
).tag(config=True)
editing_mode = Unicode(
'emacs',
help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
).tag(config=True)
autoformatter = Unicode(
None,
help=
"Autoformatter to reformat Terminal code. Can be `'black'` or `None`",
allow_none=True).tag(config=True)
mouse_support = Bool(
False,
help=
"Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
).tag(config=True)
# We don't load the list of styles for the help string, because loading
# Pygments plugins takes time and can cause unexpected errors.
highlighting_style = Union(
[Unicode('legacy'), Type(klass=Style)],
help="""The name or class of a Pygments style to use for syntax
highlighting. To see available styles, run `pygmentize -L styles`."""
).tag(config=True)
@validate('editing_mode')
def _validate_editing_mode(self, proposal):
if proposal['value'].lower() == 'vim':
proposal['value'] = 'vi'
elif proposal['value'].lower() == 'default':
proposal['value'] = 'emacs'
if hasattr(EditingMode, proposal['value'].upper()):
return proposal['value'].lower()
return self.editing_mode
@observe('editing_mode')
def _editing_mode(self, change):
if self.pt_app:
self.pt_app.editing_mode = getattr(EditingMode, change.new.upper())
@observe('autoformatter')
def _autoformatter_changed(self, change):
formatter = change.new
if formatter is None:
self.reformat_handler = lambda x: x
elif formatter == 'black':
self.reformat_handler = black_reformat_handler
else:
raise ValueError
@observe('highlighting_style')
@observe('colors')
def _highlighting_style_changed(self, change):
self.refresh_style()
def refresh_style(self):
self._style = self._make_style_from_name_or_cls(
self.highlighting_style)
highlighting_style_overrides = Dict(
help="Override highlighting format for specific tokens").tag(
config=True)
true_color = Bool(
False,
help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
"If your terminal supports true color, the following command "
"should print 'TRUECOLOR' in orange: "
"printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")).tag(
config=True)
editor = Unicode(
get_default_editor(),
help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
).tag(config=True)
prompts_class = Type(
Prompts,
help='Class used to generate Prompt token for prompt_toolkit').tag(
config=True)
prompts = Instance(Prompts)
@default('prompts')
def _prompts_default(self):
return self.prompts_class(self)
# @observe('prompts')
# def _(self, change):
# self._update_layout()
@default('displayhook_class')
def _displayhook_class_default(self):
return RichPromptDisplayHook
term_title = Bool(
True, help="Automatically set the terminal title").tag(config=True)
term_title_format = Unicode(
"IPython: {cwd}",
help=
"Customize the terminal title format. This is a python format string. "
+ "Available substitutions are: {cwd}.").tag(config=True)
display_completions = Enum(
('column', 'multicolumn', 'readlinelike'),
help=
("Options for displaying tab completions, 'column', 'multicolumn', and "
"'readlinelike'. These options are for `prompt_toolkit`, see "
"`prompt_toolkit` documentation for more information."),
default_value='multicolumn').tag(config=True)
highlight_matching_brackets = Bool(
True,
help="Highlight matching brackets.",
).tag(config=True)
extra_open_editor_shortcuts = Bool(
False,
help=
"Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
"This is in addition to the F2 binding, which is always enabled.").tag(
config=True)
handle_return = Any(
None,
help="Provide an alternative handler to be called when the user presses "
"Return. This is an advanced option intended for debugging, which "
"may be changed or removed in later releases.").tag(config=True)
enable_history_search = Bool(
True,
help="Allows to enable/disable the prompt toolkit history search").tag(
config=True)
prompt_includes_vi_mode = Bool(
True,
help="Display the current vi mode (when using vi editing mode).").tag(
config=True)
@observe('term_title')
def init_term_title(self, change=None):
# Enable or disable the terminal title.
if self.term_title:
toggle_set_term_title(True)
set_term_title(self.term_title_format.format(cwd=abbrev_cwd()))
else:
toggle_set_term_title(False)
def restore_term_title(self):
if self.term_title:
restore_term_title()
def init_display_formatter(self):
super(TerminalInteractiveShell, self).init_display_formatter()
# terminal only supports plain text
self.display_formatter.active_types = ['text/plain']
# disable `_ipython_display_`
self.display_formatter.ipython_display_formatter.enabled = False
def init_prompt_toolkit_cli(self):
if self.simple_prompt:
# Fall back to plain non-interactive output for tests.
# This is very limited.
def prompt():
prompt_text = "".join(x[1]
for x in self.prompts.in_prompt_tokens())
lines = [input(prompt_text)]
prompt_continuation = "".join(
x[1] for x in self.prompts.continuation_prompt_tokens())
while self.check_complete('\n'.join(lines))[0] == 'incomplete':
lines.append(input(prompt_continuation))
return '\n'.join(lines)
self.prompt_for_code = prompt
return
# Set up keyboard shortcuts
key_bindings = create_ipython_shortcuts(self)
# Pre-populate history from IPython's history database
history = InMemoryHistory()
last_cell = u""
for __, ___, cell in self.history_manager.get_tail(
self.history_load_length, include_latest=True):
# Ignore blank lines and consecutive duplicates
cell = cell.rstrip()
if cell and (cell != last_cell):
history.append_string(cell)
last_cell = cell
self._style = self._make_style_from_name_or_cls(
self.highlighting_style)
self.style = DynamicStyle(lambda: self._style)
editing_mode = getattr(EditingMode, self.editing_mode.upper())
self.pt_loop = asyncio.new_event_loop()
self.pt_app = PromptSession(
editing_mode=editing_mode,
key_bindings=key_bindings,
history=history,
completer=IPythonPTCompleter(shell=self),
enable_history_search=self.enable_history_search,
style=self.style,
include_default_pygments_style=False,
mouse_support=self.mouse_support,
enable_open_in_editor=self.extra_open_editor_shortcuts,
color_depth=self.color_depth,
tempfile_suffix=".py",
**self._extra_prompt_options())
def _make_style_from_name_or_cls(self, name_or_cls):
"""
Small wrapper that make an IPython compatible style from a style name
We need that to add style for prompt ... etc.
"""
style_overrides = {}
if name_or_cls == 'legacy':
legacy = self.colors.lower()
if legacy == 'linux':
style_cls = get_style_by_name('monokai')
style_overrides = _style_overrides_linux
elif legacy == 'lightbg':
style_overrides = _style_overrides_light_bg
style_cls = get_style_by_name('pastie')
elif legacy == 'neutral':
# The default theme needs to be visible on both a dark background
# and a light background, because we can't tell what the terminal
# looks like. These tweaks to the default theme help with that.
style_cls = get_style_by_name('default')
style_overrides.update({
Token.Number:
'#007700',
Token.Operator:
'noinherit',
Token.String:
'#BB6622',
Token.Name.Function:
'#2080D0',
Token.Name.Class:
'bold #2080D0',
Token.Name.Namespace:
'bold #2080D0',
Token.Name.Variable.Magic:
'#ansiblue',
Token.Prompt:
'#009900',
Token.PromptNum:
'#ansibrightgreen bold',
Token.OutPrompt:
'#990000',
Token.OutPromptNum:
'#ansibrightred bold',
})
# Hack: Due to limited color support on the Windows console
# the prompt colors will be wrong without this
if os.name == 'nt':
style_overrides.update({
Token.Prompt: '#ansidarkgreen',
Token.PromptNum: '#ansigreen bold',
Token.OutPrompt: '#ansidarkred',
Token.OutPromptNum: '#ansired bold',
})
elif legacy == 'nocolor':
style_cls = _NoStyle
style_overrides = {}
else:
raise ValueError('Got unknown colors: ', legacy)
else:
if isinstance(name_or_cls, str):
style_cls = get_style_by_name(name_or_cls)
else:
style_cls = name_or_cls
style_overrides = {
Token.Prompt: '#009900',
Token.PromptNum: '#ansibrightgreen bold',
Token.OutPrompt: '#990000',
Token.OutPromptNum: '#ansibrightred bold',
}
style_overrides.update(self.highlighting_style_overrides)
style = merge_styles([
style_from_pygments_cls(style_cls),
style_from_pygments_dict(style_overrides),
])
return style
@property
def pt_complete_style(self):
return {
'multicolumn': CompleteStyle.MULTI_COLUMN,
'column': CompleteStyle.COLUMN,
'readlinelike': CompleteStyle.READLINE_LIKE,
}[self.display_completions]
@property
def color_depth(self):
return (ColorDepth.TRUE_COLOR if self.true_color else None)
def _extra_prompt_options(self):
"""
Return the current layout option for the current Terminal InteractiveShell
"""
def get_message():
return PygmentsTokens(self.prompts.in_prompt_tokens())
if self.editing_mode == 'emacs':
# with emacs mode the prompt is (usually) static, so we call only
# the function once. With VI mode it can toggle between [ins] and
# [nor] so we can't precompute.
# here I'm going to favor the default keybinding which almost
# everybody uses to decrease CPU usage.
# if we have issues with users with custom Prompts we can see how to
# work around this.
get_message = get_message()
options = {
'complete_in_thread':
False,
'lexer':
IPythonPTLexer(),
'reserve_space_for_menu':
self.space_for_menu,
'message':
get_message,
'prompt_continuation':
(lambda width, lineno, is_soft_wrap: PygmentsTokens(
self.prompts.continuation_prompt_tokens(width))),
'multiline':
True,
'complete_style':
self.pt_complete_style,
# Highlight matching brackets, but only when this setting is
# enabled, and only when the DEFAULT_BUFFER has the focus.
'input_processors': [
ConditionalProcessor(
processor=HighlightMatchingBracketProcessor(
chars='[](){}'),
filter=HasFocus(DEFAULT_BUFFER) & ~IsDone()
& Condition(lambda: self.highlight_matching_brackets))
],
}
if not PTK3:
options['inputhook'] = self.inputhook
return options
def prompt_for_code(self):
if self.rl_next_input:
default = self.rl_next_input
self.rl_next_input = None
else:
default = ''
# In order to make sure that asyncio code written in the
# interactive shell doesn't interfere with the prompt, we run the
# prompt in a different event loop.
# If we don't do this, people could spawn coroutine with a
# while/true inside which will freeze the prompt.
try:
old_loop = asyncio.get_event_loop()
except RuntimeError:
# This happens when the user used `asyncio.run()`.
old_loop = None
asyncio.set_event_loop(self.pt_loop)
try:
with patch_stdout(raw=True):
text = self.pt_app.prompt(default=default,
**self._extra_prompt_options())
finally:
# Restore the original event loop.
asyncio.set_event_loop(old_loop)
return text
def enable_win_unicode_console(self):
# Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows
# console by default, so WUC shouldn't be needed.
from warnings import warn
warn(
"`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future",
DeprecationWarning,
stacklevel=2)
def init_io(self):
if sys.platform not in {'win32', 'cli'}:
return
import colorama
colorama.init()
# For some reason we make these wrappers around stdout/stderr.
# For now, we need to reset them so all output gets coloured.
# https://github.com/ipython/ipython/issues/8669
# io.std* are deprecated, but don't show our own deprecation warnings
# during initialization of the deprecated API.
with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
io.stdout = io.IOStream(sys.stdout)
io.stderr = io.IOStream(sys.stderr)
def init_magics(self):
super(TerminalInteractiveShell, self).init_magics()
self.register_magics(TerminalMagics)
def init_alias(self):
# The parent class defines aliases that can be safely used with any
# frontend.
super(TerminalInteractiveShell, self).init_alias()
# Now define aliases that only make sense on the terminal, because they
# need direct access to the console in a way that we can't emulate in
# GUI or web frontend
if os.name == 'posix':
for cmd in ('clear', 'more', 'less', 'man'):
self.alias_manager.soft_define_alias(cmd, cmd)
def __init__(self, *args, **kwargs):
super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
self.init_prompt_toolkit_cli()
self.init_term_title()
self.keep_running = True
self.debugger_history = InMemoryHistory()
def ask_exit(self):
self.keep_running = False
rl_next_input = None
def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
if display_banner is not DISPLAY_BANNER_DEPRECATED:
warn(
'interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.',
DeprecationWarning,
stacklevel=2)
self.keep_running = True
while self.keep_running:
print(self.separate_in, end='')
try:
code = self.prompt_for_code()
except EOFError:
if (not self.confirm_exit) \
or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
self.ask_exit()
else:
if code:
self.run_cell(code, store_history=True)
def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
# An extra layer of protection in case someone mashing Ctrl-C breaks
# out of our internal code.
if display_banner is not DISPLAY_BANNER_DEPRECATED:
warn(
'mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.',
DeprecationWarning,
stacklevel=2)
while True:
try:
self.interact()
break
except KeyboardInterrupt as e:
print("\n%s escaped interact()\n" % type(e).__name__)
finally:
# An interrupt during the eventloop will mess up the
# internal state of the prompt_toolkit library.
# Stopping the eventloop fixes this, see
# https://github.com/ipython/ipython/pull/9867
if hasattr(self, '_eventloop'):
self._eventloop.stop()
self.restore_term_title()
_inputhook = None
def inputhook(self, context):
if self._inputhook is not None:
self._inputhook(context)
active_eventloop = None
def enable_gui(self, gui=None):
if gui and (gui != 'inline'):
self.active_eventloop, self._inputhook =\
get_inputhook_name_and_func(gui)
else:
self.active_eventloop = self._inputhook = None
# For prompt_toolkit 3.0. We have to create an asyncio event loop with
# this inputhook.
if PTK3:
import asyncio
from prompt_toolkit.eventloop import new_eventloop_with_inputhook
if gui == 'asyncio':
# When we integrate the asyncio event loop, run the UI in the
# same event loop as the rest of the code. don't use an actual
# input hook. (Asyncio is not made for nesting event loops.)
self.pt_loop = asyncio.get_event_loop()
elif self._inputhook:
# If an inputhook was set, create a new asyncio event loop with
# this inputhook for the prompt.
self.pt_loop = new_eventloop_with_inputhook(self._inputhook)
else:
# When there's no inputhook, run the prompt in a separate
# asyncio event loop.
self.pt_loop = asyncio.new_event_loop()
# Run !system commands directly, not through pipes, so terminal programs
# work correctly.
system = InteractiveShell.system_raw
def auto_rewrite_input(self, cmd):
"""Overridden from the parent class to use fancy rewriting prompt"""
if not self.show_rewritten_input:
return
tokens = self.prompts.rewrite_prompt_tokens()
if self.pt_app:
print_formatted_text(PygmentsTokens(tokens),
end='',
style=self.pt_app.app.style)
print(cmd)
else:
prompt = ''.join(s for t, s in tokens)
print(prompt, cmd, sep='')
_prompts_before = None
def switch_doctest_mode(self, mode):
"""Switch prompts to classic for %doctest_mode"""
if mode:
self._prompts_before = self.prompts
self.prompts = ClassicPrompts(self)
elif self._prompts_before:
self.prompts = self._prompts_before
self._prompts_before = None
class ExecutePreprocessor(Preprocessor):
"""
Executes all the cells in a notebook
"""
timeout = Integer(30,
allow_none=True,
help=dedent("""
The time to wait (in seconds) for output from executions.
If a cell execution takes longer, an exception (TimeoutError
on python 3+, RuntimeError on python 2) is raised.
`None` or `-1` will disable the timeout. If `timeout_func` is set,
it overrides `timeout`.
""")).tag(config=True)
timeout_func = Any(default_value=None,
allow_none=True,
help=dedent("""
A callable which, when given the cell source as input,
returns the time to wait (in seconds) for output from cell
executions. If a cell execution takes longer, an exception
(TimeoutError on python 3+, RuntimeError on python 2) is
raised.
Returning `None` or `-1` will disable the timeout for the cell.
Not setting `timeout_func` will cause the preprocessor to
default to using the `timeout` trait for all cells. The
`timeout_func` trait overrides `timeout` if it is not `None`.
""")).tag(config=True)
interrupt_on_timeout = Bool(False,
help=dedent("""
If execution of a cell times out, interrupt the kernel and
continue executing other cells rather than throwing an error and
stopping.
""")).tag(config=True)
startup_timeout = Integer(60,
help=dedent("""
The time to wait (in seconds) for the kernel to start.
If kernel startup takes longer, a RuntimeError is
raised.
""")).tag(config=True)
allow_errors = Bool(False,
help=dedent("""
If `False` (default), when a cell raises an error the
execution is stopped and a `CellExecutionError`
is raised.
If `True`, execution errors are ignored and the execution
is continued until the end of the notebook. Output from
exceptions is included in the cell output in both cases.
""")).tag(config=True)
force_raise_errors = Bool(False,
help=dedent("""
If False (default), errors from executing the notebook can be
allowed with a `raises-exception` tag on a single cell, or the
`allow_errors` configurable option for all cells. An allowed error
will be recorded in notebook output, and execution will continue.
If an error occurs when it is not explicitly allowed, a
`CellExecutionError` will be raised.
If True, `CellExecutionError` will be raised for any error that occurs
while executing the notebook. This overrides both the
`allow_errors` option and the `raises-exception` cell tag.
""")).tag(config=True)
extra_arguments = List(Unicode())
kernel_name = Unicode('',
help=dedent("""
Name of kernel to use to execute the cells.
If not set, use the kernel_spec embedded in the notebook.
""")).tag(config=True)
raise_on_iopub_timeout = Bool(False,
help=dedent("""
If `False` (default), then the kernel will continue waiting for
iopub messages until it receives a kernel idle message, or until a
timeout occurs, at which point the currently executing cell will be
skipped. If `True`, then an error will be raised after the first
timeout. This option generally does not need to be used, but may be
useful in contexts where there is the possibility of executing
notebooks with memory-consuming infinite loops.
""")).tag(config=True)
store_widget_state = Bool(True,
help=dedent("""
If `True` (default), then the state of the Jupyter widgets created
at the kernel will be stored in the metadata of the notebook.
""")).tag(config=True)
iopub_timeout = Integer(4,
allow_none=False,
help=dedent("""
The time to wait (in seconds) for IOPub output. This generally
doesn't need to be set, but on some slow networks (such as CI
systems) the default timeout might not be long enough to get all
messages.
""")).tag(config=True)
shutdown_kernel = Enum(['graceful', 'immediate'],
default_value='graceful',
help=dedent("""
If `graceful` (default), then the kernel is given time to clean
up after executing all cells, e.g., to execute its `atexit` hooks.
If `immediate`, then the kernel is signaled to immediately
terminate.
""")).tag(config=True)
ipython_hist_file = Unicode(
default_value=':memory:',
help=
"""Path to file to use for SQLite history database for an IPython kernel.
The specific value `:memory:` (including the colon
at both end but not the back ticks), avoids creating a history file. Otherwise, IPython
will create a history file for each kernel.
When running kernels simultaneously (e.g. via multiprocessing) saving history a single
SQLite file can result in database errors, so using `:memory:` is recommended in non-interactive
contexts.
""").tag(config=True)
kernel_manager_class = Type(config=True,
help='The kernel manager class to use.')
@default('kernel_manager_class')
def _kernel_manager_class_default(self):
"""Use a dynamic default to avoid importing jupyter_client at startup"""
try:
from jupyter_client import KernelManager
except ImportError:
raise ImportError(
"`nbconvert --execute` requires the jupyter_client package: `pip install jupyter_client`"
)
return KernelManager
_display_id_map = Dict(help=dedent("""
mapping of locations of outputs with a given display_id
tracks cell index and output index within cell.outputs for
each appearance of the display_id
{
'display_id': {
cell_idx: [output_idx,]
}
}
"""))
def start_new_kernel(self, **kwargs):
"""Creates a new kernel manager and kernel client.
Parameters
----------
kwargs :
Any options for `self.kernel_manager_class.start_kernel()`. Because
that defaults to KernelManager, this will likely include options
accepted by `KernelManager.start_kernel()``, which includes `cwd`.
Returns
-------
km : KernelManager
A kernel manager as created by self.kernel_manager_class.
kc : KernelClient
Kernel client as created by the kernel manager `km`.
"""
if not self.kernel_name:
self.kernel_name = self.nb.metadata.get('kernelspec',
{}).get('name', 'python')
km = self.kernel_manager_class(kernel_name=self.kernel_name,
config=self.config)
if km.ipykernel and self.ipython_hist_file:
self.extra_arguments += [
'--HistoryManager.hist_file={}'.format(self.ipython_hist_file)
]
km.start_kernel(extra_arguments=self.extra_arguments, **kwargs)
kc = km.client()
kc.start_channels()
try:
kc.wait_for_ready(timeout=self.startup_timeout)
except RuntimeError:
kc.stop_channels()
km.shutdown_kernel()
raise
kc.allow_stdin = False
return km, kc
@contextmanager
def setup_preprocessor(self, nb, resources, km=None, **kwargs):
"""
Context manager for setting up the class to execute a notebook.
The assigns `nb` to `self.nb` where it will be modified in-place. It also creates
and assigns the Kernel Manager (`self.km`) and Kernel Client(`self.kc`).
It is intended to yield to a block that will execute codeself.
When control returns from the yield it stops the client's zmq channels, shuts
down the kernel, and removes the now unused attributes.
Parameters
----------
nb : NotebookNode
Notebook being executed.
resources : dictionary
Additional resources used in the conversion process. For example,
passing ``{'metadata': {'path': run_path}}`` sets the
execution path to ``run_path``.
km : KernerlManager (optional)
Optional kernel manager. If none is provided, a kernel manager will
be created.
Returns
-------
nb : NotebookNode
The executed notebook.
resources : dictionary
Additional resources used in the conversion process.
"""
path = resources.get('metadata', {}).get('path', '') or None
self.nb = nb
# clear display_id map
self._display_id_map = {}
self.widget_state = {}
self.widget_buffers = {}
if km is None:
kwargs["cwd"] = path
self.km, self.kc = self.start_new_kernel(**kwargs)
try:
# Yielding unbound args for more easier understanding and downstream consumption
yield nb, self.km, self.kc
finally:
self.kc.stop_channels()
self.km.shutdown_kernel(
now=self.shutdown_kernel == 'immediate')
for attr in ['nb', 'km', 'kc']:
delattr(self, attr)
else:
self.km = km
if not km.has_kernel:
km.start_kernel(extra_arguments=self.extra_arguments, **kwargs)
self.kc = km.client()
self.kc.start_channels()
try:
self.kc.wait_for_ready(timeout=self.startup_timeout)
except RuntimeError:
self.kc.stop_channels()
raise
self.kc.allow_stdin = False
try:
yield nb, self.km, self.kc
finally:
for attr in ['nb', 'km', 'kc']:
delattr(self, attr)
def preprocess(self, nb, resources, km=None):
"""
Preprocess notebook executing each code cell.
The input argument `nb` is modified in-place.
Parameters
----------
nb : NotebookNode
Notebook being executed.
resources : dictionary
Additional resources used in the conversion process. For example,
passing ``{'metadata': {'path': run_path}}`` sets the
execution path to ``run_path``.
km: KernelManager (optional)
Optional kernel manager. If none is provided, a kernel manager will
be created.
Returns
-------
nb : NotebookNode
The executed notebook.
resources : dictionary
Additional resources used in the conversion process.
"""
with self.setup_preprocessor(nb, resources, km=km):
self.log.info("Executing notebook with kernel: %s" %
self.kernel_name)
nb, resources = super(ExecutePreprocessor,
self).preprocess(nb, resources)
info_msg = self._wait_for_reply(self.kc.kernel_info())
nb.metadata['language_info'] = info_msg['content']['language_info']
self.set_widgets_metadata()
return nb, resources
def set_widgets_metadata(self):
if self.widget_state:
self.nb.metadata.widgets = {
'application/vnd.jupyter.widget-state+json': {
'state': {
model_id: _serialize_widget_state(state)
for model_id, state in self.widget_state.items()
if '_model_name' in state
},
'version_major': 2,
'version_minor': 0,
}
}
for key, widget in self.nb.metadata.widgets[
'application/vnd.jupyter.widget-state+json'][
'state'].items():
buffers = self.widget_buffers.get(key)
if buffers:
widget['buffers'] = buffers
def preprocess_cell(self, cell, resources, cell_index):
"""
Executes a single code cell. See base.py for details.
To execute all cells see :meth:`preprocess`.
"""
if cell.cell_type != 'code' or not cell.source.strip():
return cell, resources
reply, outputs = self.run_cell(cell, cell_index)
# Backwards compatibility for processes that wrap run_cell
cell.outputs = outputs
cell_allows_errors = (self.allow_errors
or "raises-exception" in cell.metadata.get(
"tags", []))
if self.force_raise_errors or not cell_allows_errors:
for out in cell.outputs:
if out.output_type == 'error':
raise CellExecutionError.from_cell_and_msg(cell, out)
if (reply is not None) and reply['content']['status'] == 'error':
raise CellExecutionError.from_cell_and_msg(
cell, reply['content'])
return cell, resources
def _update_display_id(self, display_id, msg):
"""Update outputs with a given display_id"""
if display_id not in self._display_id_map:
self.log.debug("display id %r not in %s", display_id,
self._display_id_map)
return
if msg['header']['msg_type'] == 'update_display_data':
msg['header']['msg_type'] = 'display_data'
try:
out = output_from_msg(msg)
except ValueError:
self.log.error("unhandled iopub msg: " + msg['msg_type'])
return
for cell_idx, output_indices in self._display_id_map[display_id].items(
):
cell = self.nb['cells'][cell_idx]
outputs = cell['outputs']
for output_idx in output_indices:
outputs[output_idx]['data'] = out['data']
outputs[output_idx]['metadata'] = out['metadata']
def _poll_for_reply(self, msg_id, cell=None, timeout=None):
try:
# check with timeout if kernel is still alive
msg = self.kc.shell_channel.get_msg(timeout=timeout)
if msg['parent_header'].get('msg_id') == msg_id:
return msg
except Empty:
# received no message, check if kernel is still alive
self._check_alive()
# kernel still alive, wait for a message
def _get_timeout(self, cell):
if self.timeout_func is not None and cell is not None:
timeout = self.timeout_func(cell)
else:
timeout = self.timeout
if not timeout or timeout < 0:
timeout = None
return timeout
def _handle_timeout(self):
self.log.error("Timeout waiting for execute reply (%is)." %
self.timeout)
if self.interrupt_on_timeout:
self.log.error("Interrupting kernel")
self.km.interrupt_kernel()
else:
raise TimeoutError("Cell execution timed out")
def _check_alive(self):
if not self.kc.is_alive():
self.log.error("Kernel died while waiting for execute reply.")
raise DeadKernelError("Kernel died")
def _wait_for_reply(self, msg_id, cell=None):
# wait for finish, with timeout
timeout = self._get_timeout(cell)
cummulative_time = 0
timeout_interval = 5
while True:
try:
msg = self.kc.shell_channel.get_msg(timeout=timeout_interval)
except Empty:
self._check_alive()
cummulative_time += timeout_interval
if timeout and cummulative_time > timeout:
self._handle_timeout()
break
else:
if msg['parent_header'].get('msg_id') == msg_id:
return msg
def _timeout_with_deadline(self, timeout, deadline):
if deadline is not None and deadline - monotonic() < timeout:
timeout = deadline - monotonic()
if timeout < 0:
timeout = 0
return timeout
def _passed_deadline(self, deadline):
if deadline is not None and deadline - monotonic() <= 0:
self._handle_timeout()
return True
return False
def run_cell(self, cell, cell_index=0):
parent_msg_id = self.kc.execute(cell.source)
self.log.debug("Executing cell:\n%s", cell.source)
exec_timeout = self._get_timeout(cell)
deadline = None
if exec_timeout is not None:
deadline = monotonic() + exec_timeout
cell.outputs = []
self.clear_before_next_output = False
# This loop resolves #659. By polling iopub_channel's and shell_channel's
# output we avoid dropping output and important signals (like idle) from
# iopub_channel. Prior to this change, iopub_channel wasn't polled until
# after exec_reply was obtained from shell_channel, leading to the
# aforementioned dropped data.
# These two variables are used to track what still needs polling:
# more_output=true => continue to poll the iopub_channel
more_output = True
# polling_exec_reply=true => continue to poll the shell_channel
polling_exec_reply = True
while more_output or polling_exec_reply:
if polling_exec_reply:
if self._passed_deadline(deadline):
polling_exec_reply = False
continue
# Avoid exceeding the execution timeout (deadline), but stop
# after at most 1s so we can poll output from iopub_channel.
timeout = self._timeout_with_deadline(1, deadline)
exec_reply = self._poll_for_reply(parent_msg_id, cell, timeout)
if exec_reply is not None:
polling_exec_reply = False
if more_output:
try:
timeout = self.iopub_timeout
if polling_exec_reply:
# Avoid exceeding the execution timeout (deadline) while
# polling for output.
timeout = self._timeout_with_deadline(
timeout, deadline)
msg = self.kc.iopub_channel.get_msg(timeout=timeout)
except Empty:
if polling_exec_reply:
# Still waiting for execution to finish so we expect that
# output may not always be produced yet.
continue
if self.raise_on_iopub_timeout:
raise TimeoutError("Timeout waiting for IOPub output")
else:
self.log.warning("Timeout waiting for IOPub output")
more_output = False
continue
if msg['parent_header'].get('msg_id') != parent_msg_id:
# not an output from our execution
continue
try:
# Will raise CellExecutionComplete when completed
self.process_message(msg, cell, cell_index)
except CellExecutionComplete:
more_output = False
# Return cell.outputs still for backwards compatibility
return exec_reply, cell.outputs
def process_message(self, msg, cell, cell_index):
"""
Processes a kernel message, updates cell state, and returns the
resulting output object that was appended to cell.outputs.
The input argument `cell` is modified in-place.
Parameters
----------
msg : dict
The kernel message being processed.
cell : nbformat.NotebookNode
The cell which is currently being processed.
cell_index : int
The position of the cell within the notebook object.
Returns
-------
output : dict
The execution output payload (or None for no output).
Raises
------
CellExecutionComplete
Once a message arrives which indicates computation completeness.
"""
msg_type = msg['msg_type']
self.log.debug("msg_type: %s", msg_type)
content = msg['content']
self.log.debug("content: %s", content)
display_id = content.get('transient', {}).get('display_id', None)
if display_id and msg_type in {
'execute_result', 'display_data', 'update_display_data'
}:
self._update_display_id(display_id, msg)
# set the prompt number for the input and the output
if 'execution_count' in content:
cell['execution_count'] = content['execution_count']
if msg_type == 'status':
if content['execution_state'] == 'idle':
raise CellExecutionComplete()
elif msg_type == 'clear_output':
self.clear_output(cell.outputs, msg, cell_index)
elif msg_type.startswith('comm'):
self.handle_comm_msg(cell.outputs, msg, cell_index)
# Check for remaining messages we don't process
elif msg_type not in ['execute_input', 'update_display_data']:
# Assign output as our processed "result"
return self.output(cell.outputs, msg, display_id, cell_index)
def output(self, outs, msg, display_id, cell_index):
msg_type = msg['msg_type']
try:
out = output_from_msg(msg)
except ValueError:
self.log.error("unhandled iopub msg: " + msg_type)
return
if self.clear_before_next_output:
self.log.debug('Executing delayed clear_output')
outs[:] = []
self.clear_display_id_mapping(cell_index)
self.clear_before_next_output = False
if display_id:
# record output index in:
# _display_id_map[display_id][cell_idx]
cell_map = self._display_id_map.setdefault(display_id, {})
output_idx_list = cell_map.setdefault(cell_index, [])
output_idx_list.append(len(outs))
outs.append(out)
return out
def clear_output(self, outs, msg, cell_index):
content = msg['content']
if content.get('wait'):
self.log.debug('Wait to clear output')
self.clear_before_next_output = True
else:
self.log.debug('Immediate clear output')
outs[:] = []
self.clear_display_id_mapping(cell_index)
def clear_display_id_mapping(self, cell_index):
for display_id, cell_map in self._display_id_map.items():
if cell_index in cell_map:
cell_map[cell_index] = []
def handle_comm_msg(self, outs, msg, cell_index):
content = msg['content']
data = content['data']
if self.store_widget_state and 'state' in data: # ignore custom msg'es
self.widget_state.setdefault(content['comm_id'],
{}).update(data['state'])
if 'buffer_paths' in data and data['buffer_paths']:
self.widget_buffers[content['comm_id']] = _get_buffer_data(msg)
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):
""" A Qt frontend for a generic Python kernel.
"""
# The text to show when the kernel is (re)started.
banner = Unicode(config=True)
kernel_banner = Unicode()
# Whether to show the banner
_display_banner = Bool(False)
# An option and corresponding signal for overriding the default kernel
# interrupt behavior.
custom_interrupt = Bool(False)
custom_interrupt_requested = QtCore.Signal()
# An option and corresponding signals for overriding the default kernel
# restart behavior.
custom_restart = Bool(False)
custom_restart_kernel_died = QtCore.Signal(float)
custom_restart_requested = QtCore.Signal()
# Whether to automatically show calltips on open-parentheses.
enable_calltips = Bool(
True,
config=True,
help="Whether to draw information calltips on open-parentheses.")
clear_on_kernel_restart = Bool(
True,
config=True,
help="Whether to clear the console when the kernel is restarted")
confirm_restart = Bool(
True,
config=True,
help="Whether to ask for user confirmation when restarting kernel")
lexer_class = DottedObjectName(config=True,
help="The pygments lexer class to use.")
is_complete_timeout = Float(
0.25,
config=True,
help="Seconds to wait for is_complete replies from the kernel.")
def _lexer_class_changed(self, name, old, new):
lexer_class = import_item(new)
self.lexer = lexer_class()
def _lexer_class_default(self):
if py3compat.PY3:
return 'pygments.lexers.Python3Lexer'
else:
return 'pygments.lexers.PythonLexer'
lexer = Any()
def _lexer_default(self):
lexer_class = import_item(self.lexer_class)
return lexer_class()
# Emitted when a user visible 'execute_request' has been submitted to the
# kernel from the FrontendWidget. Contains the code to be executed.
executing = QtCore.Signal(object)
# Emitted when a user-visible 'execute_reply' has been received from the
# kernel and processed by the FrontendWidget. Contains the response message.
executed = QtCore.Signal(object)
# Emitted when an exit request has been received from the kernel.
exit_requested = QtCore.Signal(object)
_CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos'])
_CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos'])
_ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
_local_kernel = False
_highlighter = Instance(FrontendHighlighter, allow_none=True)
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(FrontendWidget, self).__init__(*args, **kw)
# FIXME: remove this when PySide min version is updated past 1.0.7
# forcefully disable calltips if PySide is < 1.0.7, because they crash
if qt.QT_API == qt.QT_API_PYSIDE:
import PySide
if PySide.__version_info__ < (1, 0, 7):
self.log.warn(
"PySide %s < 1.0.7 detected, disabling calltips" %
PySide.__version__)
self.enable_calltips = False
# FrontendWidget protected variables.
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
self._hidden = False
self._highlighter = FrontendHighlighter(self, lexer=self.lexer)
self._kernel_manager = None
self._kernel_client = None
self._request_info = {}
self._request_info['execute'] = {}
self._callback_dict = {}
self._display_banner = True
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Configure actions.
action = self._copy_raw_action
key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C
action.setEnabled(False)
action.setShortcut(QtGui.QKeySequence(key))
action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut)
action.triggered.connect(self.copy_raw)
self.copy_available.connect(action.setEnabled)
self.addAction(action)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Set flag for whether we are connected via localhost.
self._local_kernel = kw.get('local_kernel',
FrontendWidget._local_kernel)
# Whether or not a clear_output call is pending new output.
self._pending_clearoutput = False
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
def copy(self):
""" Copy the currently selected text to the clipboard, removing prompts.
"""
if self._page_control is not None and self._page_control.hasFocus():
self._page_control.copy()
elif self._control.hasFocus():
text = self._control.textCursor().selection().toPlainText()
if text:
# Remove prompts.
lines = text.splitlines()
lines = map(transform_classic_prompt, lines)
lines = map(transform_ipy_prompt, lines)
text = '\n'.join(lines)
was_newline = text[-1] == '\n'
if was_newline: # user doesn't need newline
text = text[:-1]
QtGui.QApplication.clipboard().setText(text)
else:
self.log.debug("frontend widget : unknown copy target")
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _is_complete(self, source, interactive):
""" Returns whether 'source' can be completely processed and a new
prompt created. When triggered by an Enter/Return key press,
'interactive' is True; otherwise, it is False.
Returns
-------
(complete, indent): (bool, str)
complete is a bool, indicating whether the input is complete or not.
indent is the current indentation string for autoindent.
If complete is True, indent will be '', and should be ignored.
"""
kc = self.blocking_client
if kc is None:
self.log.warn("No blocking client to make is_complete requests")
return False, u''
msg_id = kc.is_complete(source)
while True:
try:
reply = kc.shell_channel.get_msg(
block=True, timeout=self.is_complete_timeout)
except Empty:
# assume incomplete output if we get no reply in time
return False, u''
if reply['parent_header'].get('msg_id', None) == msg_id:
status = reply['content'].get('status', u'complete')
indent = reply['content'].get('indent', u'')
return status != 'incomplete', indent
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
msg_id = self.kernel_client.execute(source, hidden)
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'user')
self._hidden = hidden
if not hidden:
self.executing.emit(source)
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _context_menu_make(self, pos):
""" Reimplemented to add an action for raw copy.
"""
menu = super(FrontendWidget, self)._context_menu_make(pos)
for before_action in menu.actions():
if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \
QtGui.QKeySequence.ExactMatch:
menu.insertAction(before_action, self._copy_raw_action)
break
return menu
def request_interrupt_kernel(self):
if self._executing:
self.interrupt_kernel()
def request_restart_kernel(self):
message = 'Are you sure you want to restart the kernel?'
self.restart_kernel(message, now=False)
def _event_filter_console_keypress(self, event):
""" Reimplemented for execution interruption and smart backspace.
"""
key = event.key()
if self._control_key_down(event.modifiers(), include_command=False):
if key == QtCore.Qt.Key_C and self._executing:
self.request_interrupt_kernel()
return True
elif key == QtCore.Qt.Key_Period:
self.request_restart_kernel()
return True
elif not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
if key == QtCore.Qt.Key_Backspace:
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(FrontendWidget,
self)._event_filter_console_keypress(event)
def _insert_continuation_prompt(self, cursor, indent=''):
""" Reimplemented for auto-indentation.
"""
super(FrontendWidget, self)._insert_continuation_prompt(cursor)
if indent:
cursor.insertText(indent)
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_clear_output(self, msg):
"""Handle clear output messages."""
if self.include_output(msg):
wait = msg['content'].get('wait', True)
if wait:
self._pending_clearoutput = True
else:
self.clear_output()
def _silent_exec_callback(self, expr, callback):
"""Silently execute `expr` in the kernel and call `callback` with reply
the `expr` is evaluated silently in the kernel (without) output in
the frontend. Call `callback` with the
`repr <http://docs.python.org/library/functions.html#repr> `_ as first argument
Parameters
----------
expr : string
valid string to be executed by the kernel.
callback : function
function accepting one argument, as a string. The string will be
the `repr` of the result of evaluating `expr`
The `callback` is called with the `repr()` of the result of `expr` as
first argument. To get the object, do `eval()` on the passed value.
See Also
--------
_handle_exec_callback : private method, deal with calling callback with reply
"""
# generate uuid, which would be used as an indication of whether or
# not the unique request originated from here (can use msg id ?)
local_uuid = str(uuid.uuid1())
msg_id = self.kernel_client.execute(
'', silent=True, user_expressions={local_uuid: expr})
self._callback_dict[local_uuid] = callback
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'silent_exec_callback')
def _handle_exec_callback(self, msg):
"""Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``
Parameters
----------
msg : raw message send by the kernel containing an `user_expressions`
and having a 'silent_exec_callback' kind.
Notes
-----
This function will look for a `callback` associated with the
corresponding message id. Association has been made by
`_silent_exec_callback`. `callback` is then called with the `repr()`
of the value of corresponding `user_expressions` as argument.
`callback` is then removed from the known list so that any message
coming again with the same id won't trigger it.
"""
user_exp = msg['content'].get('user_expressions')
if not user_exp:
return
for expression in user_exp:
if expression in self._callback_dict:
self._callback_dict.pop(expression)(user_exp[expression])
def _handle_execute_reply(self, msg):
""" Handles replies for code execution.
"""
self.log.debug("execute: %s", msg.get('content', ''))
msg_id = msg['parent_header']['msg_id']
info = self._request_info['execute'].get(msg_id)
# unset reading flag, because if execute finished, raw_input can't
# still be pending.
self._reading = False
if info and info.kind == 'user' and not self._hidden:
# Make sure that all output from the SUB channel has been processed
# before writing a new prompt.
self.kernel_client.iopub_channel.flush()
# Reset the ANSI style information to prevent bad text in stdout
# from messing up our colors. We're not a true terminal so we're
# allowed to do this.
if self.ansi_codes:
self._ansi_processor.reset_sgr()
content = msg['content']
status = content['status']
if status == 'ok':
self._process_execute_ok(msg)
elif status == 'error':
self._process_execute_error(msg)
elif status == 'aborted':
self._process_execute_abort(msg)
self._show_interpreter_prompt_for_reply(msg)
self.executed.emit(msg)
self._request_info['execute'].pop(msg_id)
elif info and info.kind == 'silent_exec_callback' and not self._hidden:
self._handle_exec_callback(msg)
self._request_info['execute'].pop(msg_id)
else:
super(FrontendWidget, self)._handle_execute_reply(msg)
def _handle_input_request(self, msg):
""" Handle requests for raw_input.
"""
self.log.debug("input: %s", msg.get('content', ''))
if self._hidden:
raise RuntimeError(
'Request for raw input during hidden execution.')
# Make sure that all output from the SUB channel has been processed
# before entering readline mode.
self.kernel_client.iopub_channel.flush()
def callback(line):
self.kernel_client.input(line)
if self._reading:
self.log.debug(
"Got second input request, assuming first was interrupted.")
self._reading = False
self._readline(msg['content']['prompt'],
callback=callback,
password=msg['content']['password'])
def _kernel_restarted_message(self, died=True):
msg = "Kernel died, restarting" if died else "Kernel restarting"
self._append_html("<br>%s<hr><br>" % msg, before_prompt=False)
def _handle_kernel_died(self, since_last_heartbeat):
"""Handle the kernel's death (if we do not own the kernel).
"""
self.log.warn("kernel died: %s", since_last_heartbeat)
if self.custom_restart:
self.custom_restart_kernel_died.emit(since_last_heartbeat)
else:
self._kernel_restarted_message(died=True)
self.reset()
def _handle_kernel_restarted(self, died=True):
"""Notice that the autorestarter restarted the kernel.
There's nothing to do but show a message.
"""
self.log.warn("kernel restarted")
self._kernel_restarted_message(died=died)
self.reset()
def _handle_inspect_reply(self, rep):
"""Handle replies for call tips."""
self.log.debug("oinfo: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('call_tip')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
content = rep['content']
if content.get('status') == 'ok' and content.get('found', False):
self._call_tip_widget.show_inspect_data(content)
def _handle_execute_result(self, msg):
""" Handle display hook output.
"""
self.log.debug("execute_result: %s", msg.get('content', ''))
if self.include_output(msg):
self.flush_clearoutput()
text = msg['content']['data']
self._append_plain_text(text + '\n', before_prompt=True)
def _handle_stream(self, msg):
""" Handle stdout, stderr, and stdin.
"""
self.log.debug("stream: %s", msg.get('content', ''))
if self.include_output(msg):
self.flush_clearoutput()
self.append_stream(msg['content']['text'])
def _handle_shutdown_reply(self, msg):
""" Handle shutdown signal, only if from other console.
"""
self.log.debug("shutdown: %s", msg.get('content', ''))
restart = msg.get('content', {}).get('restart', False)
if not self._hidden and not self.from_here(msg):
# got shutdown reply, request came from session other than ours
if restart:
# someone restarted the kernel, handle it
self._handle_kernel_restarted(died=False)
else:
# kernel was shutdown permanently
# this triggers exit_requested if the kernel was local,
# and a dialog if the kernel was remote,
# so we don't suddenly clear the qtconsole without asking.
if self._local_kernel:
self.exit_requested.emit(self)
else:
title = self.window().windowTitle()
reply = QtGui.QMessageBox.question(
self, title, "Kernel has been shutdown permanently. "
"Close the Console?", QtGui.QMessageBox.Yes,
QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
self.exit_requested.emit(self)
def _handle_status(self, msg):
"""Handle status message"""
# This is where a busy/idle indicator would be triggered,
# when we make one.
state = msg['content'].get('execution_state', '')
if state == 'starting':
# kernel started while we were running
if self._executing:
self._handle_kernel_restarted(died=True)
elif state == 'idle':
pass
elif state == 'busy':
pass
def _started_channels(self):
""" Called when the KernelManager channels have started listening or
when the frontend is assigned an already listening KernelManager.
"""
self.reset(clear=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def copy_raw(self):
""" Copy the currently selected text to the clipboard without attempting
to remove prompts or otherwise alter the text.
"""
self._control.copy()
def interrupt_kernel(self):
""" Attempts to interrupt the running kernel.
Also unsets _reading flag, to avoid runtime errors
if raw_input is called again.
"""
if self.custom_interrupt:
self._reading = False
self.custom_interrupt_requested.emit()
elif self.kernel_manager:
self._reading = False
self.kernel_manager.interrupt_kernel()
else:
self._append_plain_text(
'Cannot interrupt a kernel I did not start.\n')
def reset(self, clear=False):
""" Resets the widget to its initial state if ``clear`` parameter
is True, otherwise
prints a visual indication of the fact that the kernel restarted, but
does not clear the traces from previous usage of the kernel before it
was restarted. With ``clear=True``, it is similar to ``%clear``, but
also re-writes the banner and aborts execution if necessary.
"""
if self._executing:
self._executing = False
self._request_info['execute'] = {}
self._reading = False
self._highlighter.highlighting_on = False
if clear:
self._control.clear()
if self._display_banner:
self._append_plain_text(self.banner)
if self.kernel_banner:
self._append_plain_text(self.kernel_banner)
# update output marker for stdout/stderr, so that startup
# messages appear after banner:
self._append_before_prompt_pos = self._get_cursor().position()
self._show_interpreter_prompt()
def restart_kernel(self, message, now=False):
""" Attempts to restart the running kernel.
"""
# FIXME: now should be configurable via a checkbox in the dialog. Right
# now at least the heartbeat path sets it to True and the manual restart
# to False. But those should just be the pre-selected states of a
# checkbox that the user could override if so desired. But I don't know
# enough Qt to go implementing the checkbox now.
if self.custom_restart:
self.custom_restart_requested.emit()
return
if self.kernel_manager:
# Pause the heart beat channel to prevent further warnings.
self.kernel_client.hb_channel.pause()
# Prompt the user to restart the kernel. Un-pause the heartbeat if
# they decline. (If they accept, the heartbeat will be un-paused
# automatically when the kernel is restarted.)
if self.confirm_restart:
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
result = QtGui.QMessageBox.question(self, 'Restart kernel?',
message, buttons)
do_restart = result == QtGui.QMessageBox.Yes
else:
# confirm_restart is False, so we don't need to ask user
# anything, just do the restart
do_restart = True
if do_restart:
try:
self.kernel_manager.restart_kernel(now=now)
except RuntimeError as e:
self._append_plain_text('Error restarting kernel: %s\n' %
e,
before_prompt=True)
else:
self._append_html(
"<br>Restarting kernel...\n<hr><br>",
before_prompt=True,
)
else:
self.kernel_client.hb_channel.unpause()
else:
self._append_plain_text(
'Cannot restart a Kernel I did not start\n',
before_prompt=True)
def append_stream(self, text):
"""Appends text to the output stream."""
# Most consoles treat tabs as being 8 space characters. Convert tabs
# to spaces so that output looks as expected regardless of this
# widget's tab width.
text = text.expandtabs(8)
self._append_plain_text(text, before_prompt=True)
self._control.moveCursor(QtGui.QTextCursor.End)
def flush_clearoutput(self):
"""If a clearoutput is pending, execute it."""
if self._pending_clearoutput:
self._pending_clearoutput = False
self.clear_output()
def clear_output(self):
"""Clears the current line of output."""
cursor = self._control.textCursor()
cursor.beginEditBlock()
cursor.movePosition(cursor.StartOfLine, cursor.KeepAnchor)
cursor.insertText('')
cursor.endEditBlock()
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _auto_call_tip(self):
"""Trigger call tip automatically on open parenthesis
Call tips can be requested explcitly with `_call_tip`.
"""
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) == '(':
# trigger auto call tip on open paren
self._call_tip()
def _call_tip(self):
"""Shows a call tip, if appropriate, at the current cursor location."""
# Decide if it makes sense to show a call tip
if not self.enable_calltips or not self.kernel_client.shell_channel.is_alive(
):
return False
cursor_pos = self._get_input_buffer_cursor_pos()
code = self.input_buffer
# Send the metadata request to the kernel
msg_id = self.kernel_client.inspect(code, cursor_pos)
pos = self._get_cursor().position()
self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
return True
def _complete(self):
""" Performs completion at the current cursor location.
"""
# Send the completion request to the kernel
msg_id = self.kernel_client.complete(
code=self.input_buffer,
cursor_pos=self._get_input_buffer_cursor_pos(),
)
pos = self._get_cursor().position()
info = self._CompletionRequest(msg_id, pos)
self._request_info['complete'] = info
def _process_execute_abort(self, msg):
""" Process a reply for an aborted execution request.
"""
self._append_plain_text("ERROR: execution aborted\n")
def _process_execute_error(self, msg):
""" Process a reply for an execution request that resulted in an error.
"""
content = msg['content']
# If a SystemExit is passed along, this means exit() was called - also
# all the ipython %exit magic syntax of '-k' to be used to keep
# the kernel running
if content['ename'] == 'SystemExit':
keepkernel = content['evalue'] == '-k' or content[
'evalue'] == 'True'
self._keep_kernel_on_exit = keepkernel
self.exit_requested.emit(self)
else:
traceback = ''.join(content['traceback'])
self._append_plain_text(traceback)
def _process_execute_ok(self, msg):
""" Process a reply for a successful execution request.
"""
payload = msg['content'].get('payload', [])
for item in payload:
if not self._process_execute_payload(item):
warning = 'Warning: received unknown payload of type %s'
print(warning % repr(item['source']))
def _process_execute_payload(self, item):
""" Process a single payload item from the list of payload items in an
execution reply. Returns whether the payload was handled.
"""
# The basic FrontendWidget doesn't handle payloads, as they are a
# mechanism for going beyond the standard Python interpreter model.
return False
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self._show_prompt('>>> ')
def _show_interpreter_prompt_for_reply(self, msg):
""" Shows a prompt for the interpreter given an 'execute_reply' message.
"""
self._show_interpreter_prompt()
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._auto_call_tip()
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
""" Returns the standard Python banner.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
