class EngineFactory(RegistrationFactory):
"""IPython engine"""
# configurables:
out_stream_factory = Type('IPython.kernel.zmq.iostream.OutStream',
config=True,
help="""The OutStream for handling stdout/err.
Typically 'IPython.kernel.zmq.iostream.OutStream'""")
display_hook_factory = Type(
'IPython.kernel.zmq.displayhook.ZMQDisplayHook',
config=True,
help="""The class for handling displayhook.
Typically 'IPython.kernel.zmq.displayhook.ZMQDisplayHook'""")
location = Unicode(
config=True,
help="""The location (an IP address) of the controller. This is
used for disambiguating URLs, to determine whether
loopback should be used to connect or the public address.""")
timeout = Float(
5.0,
config=True,
help="""The time (in seconds) to wait for the Controller to respond
to registration requests before giving up.""")
max_heartbeat_misses = Integer(
50,
config=True,
help="""The maximum number of times a check for the heartbeat ping of a
controller can be missed before shutting down the engine.
If set to 0, the check is disabled.""")
sshserver = Unicode(
config=True,
help=
"""The SSH server to use for tunneling connections to the Controller."""
)
sshkey = Unicode(
config=True,
help=
"""The SSH private key file to use when tunneling connections to the Controller."""
)
paramiko = Bool(
sys.platform == 'win32',
config=True,
help="""Whether to use paramiko instead of openssh for tunnels.""")
# not configurable:
connection_info = Dict()
user_ns = Dict()
id = Integer(allow_none=True)
registrar = Instance('zmq.eventloop.zmqstream.ZMQStream')
kernel = Instance(Kernel)
hb_check_period = Integer()
# States for the heartbeat monitoring
# Initial values for monitored and pinged must satisfy "monitored > pinged == False" so that
# during the first check no "missed" ping is reported. Must be floats for Python 3 compatibility.
_hb_last_pinged = 0.0
_hb_last_monitored = 0.0
_hb_missed_beats = 0
# The zmq Stream which receives the pings from the Heart
_hb_listener = None
bident = CBytes()
ident = Unicode()
def _ident_changed(self, name, old, new):
self.bident = cast_bytes(new)
using_ssh = Bool(False)
def __init__(self, **kwargs):
super(EngineFactory, self).__init__(**kwargs)
self.ident = self.session.session
def init_connector(self):
"""construct connection function, which handles tunnels."""
self.using_ssh = bool(self.sshkey or self.sshserver)
if self.sshkey and not self.sshserver:
# We are using ssh directly to the controller, tunneling localhost to localhost
self.sshserver = self.url.split('://')[1].split(':')[0]
if self.using_ssh:
if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey,
self.paramiko):
password = False
else:
password = getpass("SSH Password for %s: " % self.sshserver)
else:
password = False
def connect(s, url):
url = disambiguate_url(url, self.location)
if self.using_ssh:
self.log.debug("Tunneling connection to %s via %s", url,
self.sshserver)
return tunnel.tunnel_connection(
s,
url,
self.sshserver,
keyfile=self.sshkey,
paramiko=self.paramiko,
password=password,
)
else:
return s.connect(url)
def maybe_tunnel(url):
"""like connect, but don't complete the connection (for use by heartbeat)"""
url = disambiguate_url(url, self.location)
if self.using_ssh:
self.log.debug("Tunneling connection to %s via %s", url,
self.sshserver)
url, tunnelobj = tunnel.open_tunnel(
url,
self.sshserver,
keyfile=self.sshkey,
paramiko=self.paramiko,
password=password,
)
return str(url)
return connect, maybe_tunnel
def register(self):
"""send the registration_request"""
self.log.info("Registering with controller at %s" % self.url)
ctx = self.context
connect, maybe_tunnel = self.init_connector()
reg = ctx.socket(zmq.DEALER)
reg.setsockopt(zmq.IDENTITY, self.bident)
connect(reg, self.url)
self.registrar = zmqstream.ZMQStream(reg, self.loop)
content = dict(uuid=self.ident)
self.registrar.on_recv(
lambda msg: self.complete_registration(msg, connect, maybe_tunnel))
# print (self.session.key)
self.session.send(self.registrar,
"registration_request",
content=content)
def _report_ping(self, msg):
"""Callback for when the heartmonitor.Heart receives a ping"""
#self.log.debug("Received a ping: %s", msg)
self._hb_last_pinged = time.time()
def complete_registration(self, msg, connect, maybe_tunnel):
# print msg
self._abort_dc.stop()
ctx = self.context
loop = self.loop
identity = self.bident
idents, msg = self.session.feed_identities(msg)
msg = self.session.unserialize(msg)
content = msg['content']
info = self.connection_info
def url(key):
"""get zmq url for given channel"""
return str(info["interface"] + ":%i" % info[key])
if content['status'] == 'ok':
self.id = int(content['id'])
# launch heartbeat
# possibly forward hb ports with tunnels
hb_ping = maybe_tunnel(url('hb_ping'))
hb_pong = maybe_tunnel(url('hb_pong'))
hb_monitor = None
if self.max_heartbeat_misses > 0:
# Add a monitor socket which will record the last time a ping was seen
mon = self.context.socket(zmq.SUB)
mport = mon.bind_to_random_port('tcp://%s' % LOCALHOST)
mon.setsockopt(zmq.SUBSCRIBE, b"")
self._hb_listener = zmqstream.ZMQStream(mon, self.loop)
self._hb_listener.on_recv(self._report_ping)
hb_monitor = "tcp://%s:%i" % (LOCALHOST, mport)
heart = Heart(hb_ping, hb_pong, hb_monitor, heart_id=identity)
heart.start()
# create Shell Connections (MUX, Task, etc.):
shell_addrs = url('mux'), url('task')
# Use only one shell stream for mux and tasks
stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
stream.setsockopt(zmq.IDENTITY, identity)
shell_streams = [stream]
for addr in shell_addrs:
connect(stream, addr)
# control stream:
control_addr = url('control')
control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop)
control_stream.setsockopt(zmq.IDENTITY, identity)
connect(control_stream, control_addr)
# create iopub stream:
iopub_addr = url('iopub')
iopub_socket = ctx.socket(zmq.PUB)
iopub_socket.setsockopt(zmq.IDENTITY, identity)
connect(iopub_socket, iopub_addr)
# disable history:
self.config.HistoryManager.hist_file = ':memory:'
# Redirect input streams and set a display hook.
if self.out_stream_factory:
sys.stdout = self.out_stream_factory(self.session,
iopub_socket, u'stdout')
sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id)
sys.stderr = self.out_stream_factory(self.session,
iopub_socket, u'stderr')
sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id)
if self.display_hook_factory:
sys.displayhook = self.display_hook_factory(
self.session, iopub_socket)
sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id)
self.kernel = Kernel(parent=self,
int_id=self.id,
ident=self.ident,
session=self.session,
control_stream=control_stream,
shell_streams=shell_streams,
iopub_socket=iopub_socket,
loop=loop,
user_ns=self.user_ns,
log=self.log)
self.kernel.shell.display_pub.topic = cast_bytes(
'engine.%i.displaypub' % self.id)
# periodically check the heartbeat pings of the controller
# Should be started here and not in "start()" so that the right period can be taken
# from the hubs HeartBeatMonitor.period
if self.max_heartbeat_misses > 0:
# Use a slightly bigger check period than the hub signal period to not warn unnecessary
self.hb_check_period = int(content['hb_period']) + 10
self.log.info(
"Starting to monitor the heartbeat signal from the hub every %i ms.",
self.hb_check_period)
self._hb_reporter = ioloop.PeriodicCallback(
self._hb_monitor, self.hb_check_period, self.loop)
self._hb_reporter.start()
else:
self.log.info(
"Monitoring of the heartbeat signal from the hub is not enabled."
)
# FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged
app = IPKernelApp(parent=self,
shell=self.kernel.shell,
kernel=self.kernel,
log=self.log)
app.init_profile_dir()
app.init_code()
self.kernel.start()
else:
self.log.fatal("Registration Failed: %s" % msg)
raise Exception("Registration Failed: %s" % msg)
self.log.info("Completed registration with id %i" % self.id)
def abort(self):
self.log.fatal("Registration timed out after %.1f seconds" %
self.timeout)
if self.url.startswith('127.'):
self.log.fatal("""
If the controller and engines are not on the same machine,
you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py):
c.HubFactory.ip='*' # for all interfaces, internal and external
c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see
or tunnel connections via ssh.
""")
self.session.send(self.registrar,
"unregistration_request",
content=dict(id=self.id))
time.sleep(1)
sys.exit(255)
def _hb_monitor(self):
"""Callback to monitor the heartbeat from the controller"""
self._hb_listener.flush()
if self._hb_last_monitored > self._hb_last_pinged:
self._hb_missed_beats += 1
self.log.warn(
"No heartbeat in the last %s ms (%s time(s) in a row).",
self.hb_check_period, self._hb_missed_beats)
else:
#self.log.debug("Heartbeat received (after missing %s beats).", self._hb_missed_beats)
self._hb_missed_beats = 0
if self._hb_missed_beats >= self.max_heartbeat_misses:
self.log.fatal(
"Maximum number of heartbeats misses reached (%s times %s ms), shutting down.",
self.max_heartbeat_misses, self.hb_check_period)
self.session.send(self.registrar,
"unregistration_request",
content=dict(id=self.id))
self.loop.stop()
self._hb_last_monitored = time.time()
def start(self):
dc = ioloop.DelayedCallback(self.register, 0, self.loop)
dc.start()
self._abort_dc = ioloop.DelayedCallback(self.abort,
self.timeout * 1000, self.loop)
self._abort_dc.start()
class NbGraderApp(BaseNbGraderApp):
name = 'nbgrader'
description = u'A system for assigning and grading notebooks'
version = u'0.2.0'
aliases = aliases
flags = flags
examples = """
The nbgrader application is a system for assigning and grading notebooks.
Each subcommand of this program corresponds to a different step in the
grading process. In order to facilitate the grading pipeline, nbgrader
places some constraints on how the assignments must be structured. By
default, the directory structure for the assignments must look like this:
{nbgrader_step}/{student_id}/{assignment_id}/{notebook_id}.ipynb
where 'nbgrader_step' is the step in the nbgrader pipeline, 'student_id'
is the ID of the student, 'assignment_id' is the name of the assignment,
and 'notebook_id' is the name of the notebook (excluding the extension).
For example, when running `nbgrader autograde "Problem Set 1"`, the
autograder will first look for all notebooks for all students in the
following directories:
submitted/*/Problem Set 1/*.ipynb
and it will write the autograded notebooks to the corresponding directory
and filename for each notebook and each student:
autograded/{student_id}/Problem Set 1/{notebook_id}.ipynb
These variables, as well as the overall directory structure, can be
configured through the `NbGraderConfig` class (run `nbgrader --help-all`
to see these options).
For more details on how each of the subcommands work, please see the help
for that command (e.g. `nbgrader assign --help-all`).
"""
subcommands = dict(assign=(AssignApp,
dedent("""
Create the student version of an assignment. Intended for use by
instructors only.
""").strip()),
autograde=(AutogradeApp,
dedent("""
Autograde submitted assignments. Intended for use by instructors
only.
""").strip()),
formgrade=(FormgradeApp,
dedent("""
Manually grade assignments (after autograding). Intended for use
by instructors only.
""").strip()),
feedback=(FeedbackApp,
dedent("""
Generate feedback (after autograding and manual grading).
Intended for use by instructors only.
""").strip()),
validate=(ValidateApp,
dedent("""
Validate a notebook in an assignment. Intended for use by
instructors and students.
""").strip()),
release=(ReleaseApp,
dedent("""
Release an assignment to students through the nbgrader exchange.
Intended for use by instructors only.
""").strip()),
collect=(CollectApp,
dedent("""
Collect an assignment from students through the nbgrader exchange.
Intended for use by instructors only.
""").strip()),
fetch=(FetchApp,
dedent("""
Fetch an assignment from an instructor through the nbgrader exchange.
Intended for use by students only.
""").strip()),
submit=(SubmitApp,
dedent("""
Submit an assignment to an instructor through the nbgrader exchange.
Intended for use by students only.
""").strip()),
list=(ListApp,
dedent("""
List inbound or outbound assignments in the nbgrader exchange.
Intended for use by instructors and students.
""").strip()),
extension=(ExtensionApp,
dedent("""
Install and activate the "Create Assignment" notebook extension.
""").strip()))
generate_config = Bool(False,
config=True,
help="Generate a new config file")
def _classes_default(self):
classes = super(NbGraderApp, self)._classes_default()
# include all the apps that have configurable options
for appname, (app, help) in self.subcommands.items():
if len(app.class_traits(config=True)) > 0:
classes.append(app)
# include all preprocessors that have configurable options
for pp_name in preprocessors.__all__:
pp = getattr(preprocessors, pp_name)
if len(pp.class_traits(config=True)) > 0:
classes.append(pp)
return classes
@catch_config_error
def initialize(self, argv=None):
super(NbGraderApp, self).initialize(argv)
def start(self):
# if we're generating a config file, then do only that
if self.generate_config:
s = self.generate_config_file()
filename = "nbgrader_config.py"
if os.path.exists(filename) and not self.overwrite:
self.fail(
"Config file '{}' already exists (run with --overwrite to overwrite it)"
.format(filename))
with open(filename, 'w') as fh:
fh.write(s)
self.log.info("New config file saved to '{}'".format(filename))
sys.exit(0)
# check: is there a subapp given?
if self.subapp is None:
self.fail("No command given (run with --help for options)")
# This starts subapps
super(NbGraderApp, self).start()
class Kernel(SingletonConfigurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
def _eventloop_changed(self, name, old, new):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
session = Instance(Session, allow_none=True)
profile_dir = Instance('IPython.core.profiledir.ProfileDir',
allow_none=True)
shell_streams = List()
control_stream = Instance(ZMQStream, allow_none=True)
iopub_socket = Instance(zmq.Socket, allow_none=True)
stdin_socket = Instance(zmq.Socket, allow_none=True)
log = Instance(logging.Logger, allow_none=True)
# identities:
int_id = Integer(-1)
ident = Unicode()
def _ident_default(self):
return unicode_type(uuid.uuid4())
# This should be overridden by wrapper kernels that implement any real
# language.
language_info = {}
# any links that should go in the help menu
help_links = List()
# Private interface
_darwin_app_nap = Bool(
True,
config=True,
help="""Whether to use appnope for compatiblity with OS X App Nap.
Only affects OS X >= 10.9.
""")
# track associations with current request
_allow_stdin = Bool(False)
_parent_header = Dict()
_parent_ident = Any(b'')
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005, config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05, config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# set of aborted msg_ids
aborted = Set()
# Track execution count here. For IPython, we override this to use the
# execution count we store in the shell.
execution_count = 0
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Build dict of handlers for message types
msg_types = [
'execute_request',
'complete_request',
'inspect_request',
'history_request',
'kernel_info_request',
'connect_request',
'shutdown_request',
'apply_request',
'is_complete_request',
]
self.shell_handlers = {}
for msg_type in msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
control_msg_types = msg_types + ['clear_request', 'abort_request']
self.control_handlers = {}
for msg_type in control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.deserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
# Set the parent message for side effects.
self.set_parent(idents, msg)
self._publish_status(u'busy')
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if msg_id in self.aborted:
self.aborted.remove(msg_id)
# is it safe to assume a msg_id will not be resubmitted?
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
# ensure default_int_handler during handler call
sig = signal(SIGINT, default_int_handler)
self.log.debug("%s: %s", msg_type, msg)
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
signal(SIGINT, sig)
sys.stdout.flush()
sys.stderr.flush()
self._publish_status(u'idle')
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop %s", self.eventloop)
for stream in self.shell_streams:
# flush any pending replies,
# which may be skipped by entering the eventloop
stream.flush(zmq.POLLOUT)
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
# publish idle status
self._publish_status('starting')
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _make_metadata(self, other=None):
"""init metadata dict, for execute/apply_reply"""
new_md = {
'dependencies_met': True,
'engine': self.ident,
'started': datetime.now(),
}
if other:
new_md.update(other)
return new_md
def _publish_execute_input(self, code, parent, execution_count):
"""Publish the code request on the iopub stream."""
self.session.send(self.iopub_socket,
u'execute_input', {
u'code': code,
u'execution_count': execution_count
},
parent=parent,
ident=self._topic('execute_input'))
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(
self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent or self._parent_header,
ident=self._topic('status'),
)
def set_parent(self, ident, parent):
"""Set the current parent_header
Side effects (IOPub messages) and replies are associated with
the request that caused them via the parent_header.
The parent identity is used to route input_request messages
on the stdin channel.
"""
self._parent_ident = ident
self._parent_header = parent
def send_response(self,
stream,
msg_or_type,
content=None,
ident=None,
buffers=None,
track=False,
header=None,
metadata=None):
"""Send a response to the message we're currently processing.
This accepts all the parameters of :meth:`ipython_kernel.session.Session.send`
except ``parent``.
This relies on :meth:`set_parent` having been called for the current
message.
"""
return self.session.send(stream, msg_or_type, content,
self._parent_header, ident, buffers, track,
header, metadata)
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
try:
content = parent[u'content']
code = py3compat.cast_unicode_py2(content[u'code'])
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
user_expressions = content.get('user_expressions', {})
allow_stdin = content.get('allow_stdin', False)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
stop_on_error = content.get('stop_on_error', True)
md = self._make_metadata(parent['metadata'])
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self.execution_count += 1
self._publish_execute_input(code, parent, self.execution_count)
reply_content = self.do_execute(code, silent, store_history,
user_expressions, allow_stdin)
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
md['status'] = reply_content['status']
if reply_content['status'] == 'error' and \
reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent,
metadata=md,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content'][
'status'] == u'error' and stop_on_error:
self._abort_queues()
def do_execute(self,
code,
silent,
store_history=True,
user_expressions=None,
allow_stdin=False):
"""Execute user code. Must be overridden by subclasses.
"""
raise NotImplementedError
def complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
cursor_pos = content['cursor_pos']
matches = self.do_complete(code, cursor_pos)
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply', matches,
parent, ident)
self.log.debug("%s", completion_msg)
def do_complete(self, code, cursor_pos):
"""Override in subclasses to find completions.
"""
return {
'matches': [],
'cursor_end': cursor_pos,
'cursor_start': cursor_pos,
'metadata': {},
'status': 'ok'
}
def inspect_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_inspect(content['code'], content['cursor_pos'],
content.get('detail_level', 0))
# Before we send this object over, we scrub it for JSON usage
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'inspect_reply', reply_content, parent,
ident)
self.log.debug("%s", msg)
def do_inspect(self, code, cursor_pos, detail_level=0):
"""Override in subclasses to allow introspection.
"""
return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False}
def history_request(self, stream, ident, parent):
content = parent['content']
reply_content = self.do_history(**content)
reply_content = json_clean(reply_content)
msg = self.session.send(stream, 'history_reply', reply_content, parent,
ident)
self.log.debug("%s", msg)
def do_history(self,
hist_access_type,
output,
raw,
session=None,
start=None,
stop=None,
n=None,
pattern=None,
unique=False):
"""Override in subclasses to access history.
"""
return {'history': []}
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
msg = self.session.send(stream, 'connect_reply', content, parent,
ident)
self.log.debug("%s", msg)
@property
def kernel_info(self):
return {
'protocol_version': release.kernel_protocol_version,
'implementation': self.implementation,
'implementation_version': self.implementation_version,
'language_info': self.language_info,
'banner': self.banner,
'help_links': self.help_links,
}
def kernel_info_request(self, stream, ident, parent):
msg = self.session.send(stream, 'kernel_info_reply', self.kernel_info,
parent, ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
content = self.do_shutdown(parent['content']['restart'])
self.session.send(stream,
u'shutdown_reply',
content,
parent,
ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply', content,
parent)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time() + 0.1, loop.stop)
def do_shutdown(self, restart):
"""Override in subclasses to do things when the frontend shuts down the
kernel.
"""
return {'status': 'ok', 'restart': restart}
def is_complete_request(self, stream, ident, parent):
content = parent['content']
code = content['code']
reply_content = self.do_is_complete(code)
reply_content = json_clean(reply_content)
reply_msg = self.session.send(stream, 'is_complete_reply',
reply_content, parent, ident)
self.log.debug("%s", reply_msg)
def do_is_complete(self, code):
"""Override in subclasses to find completions.
"""
return {
'status': 'unknown',
}
#---------------------------------------------------------------------------
# Engine methods
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
md = self._make_metadata(parent['metadata'])
reply_content, result_buf = self.do_apply(content, bufs, msg_id, md)
# put 'ok'/'error' status in header, for scheduler introspection:
md['status'] = reply_content['status']
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
self.session.send(stream,
u'apply_reply',
reply_content,
parent=parent,
ident=ident,
buffers=result_buf,
metadata=md)
def do_apply(self, content, bufs, msg_id, reply_metadata):
"""Override in subclasses to support the IPython parallel framework.
"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Control messages
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specific msg by id"""
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, string_types):
msg_ids = [msg_ids]
if not msg_ids:
self._abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream,
'abort_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
content = self.do_clear()
self.session.send(stream,
'clear_reply',
ident=idents,
parent=parent,
content=content)
def do_clear(self):
"""Override in subclasses to clear the namespace
This is only required for IPython.parallel.
"""
raise NotImplementedError
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
if self.int_id >= 0:
base = "engine.%i" % self.int_id
else:
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def getpass(self, prompt=''):
"""Forward getpass to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"getpass was called, but this frontend does not support input requests."
)
return self._input_request(
prompt,
self._parent_ident,
self._parent_header,
password=True,
)
def raw_input(self, prompt=''):
"""Forward raw_input to frontends
Raises
------
StdinNotImplentedError if active frontend doesn't support stdin.
"""
if not self._allow_stdin:
raise StdinNotImplementedError(
"raw_input was called, but this frontend does not support input requests."
)
return self._input_request(
prompt,
self._parent_ident,
self._parent_header,
password=False,
)
def _input_request(self, prompt, ident, parent, password=False):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# flush the stdin socket, to purge stale replies
while True:
try:
self.stdin_socket.recv_multipart(zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
break
else:
raise
# Send the input request.
content = json_clean(dict(prompt=prompt, password=password))
self.session.send(self.stdin_socket,
u'input_request',
content,
parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
except KeyboardInterrupt:
# re-raise KeyboardInterrupt, to truncate traceback
raise KeyboardInterrupt
else:
break
try:
value = py3compat.unicode_to_str(reply['content']['value'])
except:
self.log.error("Bad input_reply: %s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket,
self._shutdown_message,
ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[s.flush(zmq.POLLOUT) for s in self.shell_streams]
class Kernel(Configurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
def _eventloop_changed(self, name, old, new):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_callback(self.enter_eventloop)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
shell_class = Type(ZMQInteractiveShell)
session = Instance(Session)
profile_dir = Instance('IPython.core.profiledir.ProfileDir')
shell_streams = List()
control_stream = Instance(ZMQStream)
iopub_socket = Instance(zmq.Socket)
stdin_socket = Instance(zmq.Socket)
log = Instance(logging.Logger)
user_module = Any()
def _user_module_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_module = new
user_ns = Instance(dict, args=None, allow_none=True)
def _user_ns_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_ns = new
self.shell.init_user_ns()
# identities:
int_id = Integer(-1)
ident = Unicode()
def _ident_default(self):
return unicode_type(uuid.uuid4())
# Private interface
_darwin_app_nap = Bool(
True,
config=True,
help="""Whether to use appnope for compatiblity with OS X App Nap.
Only affects OS X >= 10.9.
""")
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005, config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05, config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# A reference to the Python builtin 'raw_input' function.
# (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
_sys_raw_input = Any()
_sys_eval_input = Any()
# set of aborted msg_ids
aborted = Set()
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Initialize the InteractiveShell subclass
self.shell = self.shell_class.instance(
parent=self,
profile_dir=self.profile_dir,
user_module=self.user_module,
user_ns=self.user_ns,
kernel=self,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('pyout')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.shell.data_pub.session = self.session
self.shell.data_pub.pub_socket = self.iopub_socket
# TMP - hack while developing
self.shell._reply_content = None
# Build dict of handlers for message types
msg_types = [
'execute_request',
'complete_request',
'object_info_request',
'history_request',
'kernel_info_request',
'connect_request',
'shutdown_request',
'apply_request',
]
self.shell_handlers = {}
for msg_type in msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
comm_msg_types = ['comm_open', 'comm_msg', 'comm_close']
comm_manager = self.shell.comm_manager
for msg_type in comm_msg_types:
self.shell_handlers[msg_type] = getattr(comm_manager, msg_type)
control_msg_types = msg_types + ['clear_request', 'abort_request']
self.control_handlers = {}
for msg_type in control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
header = msg['header']
msg_id = header['msg_id']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents, msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if msg_id in self.aborted:
self.aborted.remove(msg_id)
# is it safe to assume a msg_id will not be resubmitted?
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
# ensure default_int_handler during handler call
sig = signal(SIGINT, default_int_handler)
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
signal(SIGINT, sig)
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop %s", self.eventloop)
for stream in self.shell_streams:
# flush any pending replies,
# which may be skipped by entering the eventloop
stream.flush(zmq.POLLOUT)
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
self.shell.exit_now = False
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
# publish idle status
self._publish_status('starting')
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _make_metadata(self, other=None):
"""init metadata dict, for execute/apply_reply"""
new_md = {
'dependencies_met': True,
'engine': self.ident,
'started': datetime.now(),
}
if other:
new_md.update(other)
return new_md
def _publish_pyin(self, code, parent, execution_count):
"""Publish the code request on the pyin stream."""
self.session.send(self.iopub_socket,
u'pyin', {
u'code': code,
u'execution_count': execution_count
},
parent=parent,
ident=self._topic('pyin'))
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(
self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent,
ident=self._topic('status'),
)
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
self._publish_status(u'busy', parent)
try:
content = parent[u'content']
code = py3compat.cast_unicode_py2(content[u'code'])
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
md = self._make_metadata(parent['metadata'])
shell = self.shell # we'll need this a lot here
# Replace raw_input. Note that is not sufficient to replace
# raw_input in the user namespace.
if content.get('allow_stdin', False):
raw_input = lambda prompt='': self._raw_input(
prompt, ident, parent)
input = lambda prompt='': eval(raw_input(prompt))
else:
raw_input = input = lambda prompt='': self._no_raw_input()
if py3compat.PY3:
self._sys_raw_input = builtin_mod.input
builtin_mod.input = raw_input
else:
self._sys_raw_input = builtin_mod.raw_input
self._sys_eval_input = builtin_mod.input
builtin_mod.raw_input = raw_input
builtin_mod.input = input
# Set the parent message of the display hook and out streams.
shell.set_parent(parent)
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self._publish_pyin(code, parent, shell.execution_count)
reply_content = {}
try:
# FIXME: the shell calls the exception handler itself.
shell.run_cell(code, store_history=store_history, silent=silent)
except:
status = u'error'
# FIXME: this code right now isn't being used yet by default,
# because the run_cell() call above directly fires off exception
# reporting. This code, therefore, is only active in the scenario
# where runlines itself has an unhandled exception. We need to
# uniformize this, for all exception construction to come from a
# single location in the codbase.
etype, evalue, tb = sys.exc_info()
tb_list = traceback.format_exception(etype, evalue, tb)
reply_content.update(shell._showtraceback(etype, evalue, tb_list))
else:
status = u'ok'
finally:
# Restore raw_input.
if py3compat.PY3:
builtin_mod.input = self._sys_raw_input
else:
builtin_mod.raw_input = self._sys_raw_input
builtin_mod.input = self._sys_eval_input
reply_content[u'status'] = status
# Return the execution counter so clients can display prompts
reply_content['execution_count'] = shell.execution_count - 1
# FIXME - fish exception info out of shell, possibly left there by
# runlines. We'll need to clean up this logic later.
if shell._reply_content is not None:
reply_content.update(shell._reply_content)
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method='execute')
reply_content['engine_info'] = e_info
# reset after use
shell._reply_content = None
if 'traceback' in reply_content:
self.log.info("Exception in execute request:\n%s",
'\n'.join(reply_content['traceback']))
# At this point, we can tell whether the main code execution succeeded
# or not. If it did, we proceed to evaluate user_variables/expressions
if reply_content['status'] == 'ok':
reply_content[u'user_variables'] = \
shell.user_variables(content.get(u'user_variables', []))
reply_content[u'user_expressions'] = \
shell.user_expressions(content.get(u'user_expressions', {}))
else:
# If there was an error, don't even try to compute variables or
# expressions
reply_content[u'user_variables'] = {}
reply_content[u'user_expressions'] = {}
# Payloads should be retrieved regardless of outcome, so we can both
# recover partial output (that could have been generated early in a
# block, before an error) and clear the payload system always.
reply_content[u'payload'] = shell.payload_manager.read_payload()
# Be agressive about clearing the payload because we don't want
# it to sit in memory until the next execute_request comes in.
shell.payload_manager.clear_payload()
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
md['status'] = reply_content['status']
if reply_content['status'] == 'error' and \
reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
reply_msg = self.session.send(stream,
u'execute_reply',
reply_content,
parent,
metadata=md,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content']['status'] == u'error':
self._abort_queues()
self._publish_status(u'idle', parent)
def complete_request(self, stream, ident, parent):
txt, matches = self._complete(parent)
matches = {'matches': matches, 'matched_text': txt, 'status': 'ok'}
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply', matches,
parent, ident)
self.log.debug("%s", completion_msg)
def object_info_request(self, stream, ident, parent):
content = parent['content']
object_info = self.shell.object_inspect(content['oname'],
detail_level=content.get(
'detail_level', 0))
# Before we send this object over, we scrub it for JSON usage
oinfo = json_clean(object_info)
msg = self.session.send(stream, 'object_info_reply', oinfo, parent,
ident)
self.log.debug("%s", msg)
def history_request(self, stream, ident, parent):
# We need to pull these out, as passing **kwargs doesn't work with
# unicode keys before Python 2.6.5.
hist_access_type = parent['content']['hist_access_type']
raw = parent['content']['raw']
output = parent['content']['output']
if hist_access_type == 'tail':
n = parent['content']['n']
hist = self.shell.history_manager.get_tail(n,
raw=raw,
output=output,
include_latest=True)
elif hist_access_type == 'range':
session = parent['content']['session']
start = parent['content']['start']
stop = parent['content']['stop']
hist = self.shell.history_manager.get_range(session,
start,
stop,
raw=raw,
output=output)
elif hist_access_type == 'search':
n = parent['content'].get('n')
unique = parent['content'].get('unique', False)
pattern = parent['content']['pattern']
hist = self.shell.history_manager.search(pattern,
raw=raw,
output=output,
n=n,
unique=unique)
else:
hist = []
hist = list(hist)
content = {'history': hist}
content = json_clean(content)
msg = self.session.send(stream, 'history_reply', content, parent,
ident)
self.log.debug("Sending history reply with %i entries", len(hist))
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
msg = self.session.send(stream, 'connect_reply', content, parent,
ident)
self.log.debug("%s", msg)
def kernel_info_request(self, stream, ident, parent):
vinfo = {
'protocol_version': protocol_version,
'ipython_version': ipython_version,
'language_version': language_version,
'language': 'python',
}
msg = self.session.send(stream, 'kernel_info_reply', vinfo, parent,
ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
self.shell.exit_now = True
content = dict(status='ok')
content.update(parent['content'])
self.session.send(stream,
u'shutdown_reply',
content,
parent,
ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply', content,
parent)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time() + 0.1, loop.stop)
#---------------------------------------------------------------------------
# Engine methods
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
self._publish_status(u'busy', parent)
# Set the parent message of the display hook and out streams.
shell = self.shell
shell.set_parent(parent)
# pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
# self.iopub_socket.send(pyin_msg)
# self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
md = self._make_metadata(parent['metadata'])
try:
working = shell.user_ns
prefix = "_" + str(msg_id).replace("-", "") + "_"
f, args, kwargs = unpack_apply_message(bufs, working, copy=False)
fname = getattr(f, '__name__', 'f')
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
ns = {fname: f, argname: args, kwargname: kwargs, resultname: None}
# print ns
working.update(ns)
code = "%s = %s(*%s,**%s)" % (resultname, fname, argname,
kwargname)
try:
exec(code, shell.user_global_ns, shell.user_ns)
result = working.get(resultname)
finally:
for key in ns:
working.pop(key)
result_buf = serialize_object(
result,
buffer_threshold=self.session.buffer_threshold,
item_threshold=self.session.item_threshold,
)
except:
# invoke IPython traceback formatting
shell.showtraceback()
# FIXME - fish exception info out of shell, possibly left there by
# run_code. We'll need to clean up this logic later.
reply_content = {}
if shell._reply_content is not None:
reply_content.update(shell._reply_content)
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method='apply')
reply_content['engine_info'] = e_info
# reset after use
shell._reply_content = None
self.session.send(self.iopub_socket,
u'pyerr',
reply_content,
parent=parent,
ident=self._topic('pyerr'))
self.log.info("Exception in apply request:\n%s",
'\n'.join(reply_content['traceback']))
result_buf = []
if reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
else:
reply_content = {'status': 'ok'}
# put 'ok'/'error' status in header, for scheduler introspection:
md['status'] = reply_content['status']
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
reply_msg = self.session.send(stream,
u'apply_reply',
reply_content,
parent=parent,
ident=ident,
buffers=result_buf,
metadata=md)
self._publish_status(u'idle', parent)
#---------------------------------------------------------------------------
# Control messages
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specifig msg by id"""
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, string_types):
msg_ids = [msg_ids]
if not msg_ids:
self.abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream,
'abort_reply',
content=content,
parent=parent,
ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
self.shell.reset(False)
msg = self.session.send(stream,
'clear_reply',
ident=idents,
parent=parent,
content=dict(status='ok'))
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _wrap_exception(self, method=None):
# import here, because _wrap_exception is only used in parallel,
# and parallel has higher min pyzmq version
from IPython.parallel.error import wrap_exception
e_info = dict(engine_uuid=self.ident,
engine_id=self.int_id,
method=method)
content = wrap_exception(e_info)
return content
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
if self.int_id >= 0:
base = "engine.%i" % self.int_id
else:
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status': 'aborted'}
md = {'engine': self.ident}
md.update(status)
reply_msg = self.session.send(stream,
reply_type,
metadata=md,
content=status,
parent=msg,
ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def _raw_input(self, prompt, ident, parent):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# flush the stdin socket, to purge stale replies
while True:
try:
self.stdin_socket.recv_multipart(zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
break
else:
raise
# Send the input request.
content = json_clean(dict(prompt=prompt))
self.session.send(self.stdin_socket,
u'input_request',
content,
parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
except KeyboardInterrupt:
# re-raise KeyboardInterrupt, to truncate traceback
raise KeyboardInterrupt
else:
break
try:
value = py3compat.unicode_to_str(reply['content']['value'])
except:
self.log.error("Got bad raw_input reply: ")
self.log.error("%s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _complete(self, msg):
c = msg['content']
try:
cpos = int(c['cursor_pos'])
except:
# If we don't get something that we can convert to an integer, at
# least attempt the completion guessing the cursor is at the end of
# the text, if there's any, and otherwise of the line
cpos = len(c['text'])
if cpos == 0:
cpos = len(c['line'])
return self.shell.complete(c['text'], c['line'], cpos)
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket,
self._shutdown_message,
ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[s.flush(zmq.POLLOUT) for s in self.shell_streams]
class _IntWidget(DOMWidget):
value = CInt(0, help="Int value", sync=True)
disabled = Bool(False, help="Enable or disable user changes", sync=True)
description = Unicode(
help="Description of the value this widget represents", sync=True)
class ConverterTemplate(Configurable):
""" A Jinja2 base converter templates
Preprocess the ipynb files, feed it throug jinja templates,
and spit an converted files and a data object with other data
should be mostly configurable
"""
pre_transformer_order = List(['haspyout_transformer'],
config=True,
help="""
An ordered list of pre transformer to apply to the ipynb
file before running through templates
""")
tex_environement = Bool(False,
config=True,
help=""" is this a tex environment or not """)
template_file = Unicode('',
config=True,
help=""" Name of the template file to use """)
#-------------------------------------------------------------------------
# Instance-level attributes that are set in the constructor for this
# class.
#-------------------------------------------------------------------------
preprocessors = []
def __init__(self, preprocessors={}, jinja_filters={}, config=None, **kw):
""" Init a new converter.
config: the Configurable config object to pass around.
preprocessors: dict of **availlable** key/value function to run on
ipynb json data before conversion to extract/inline file.
See `transformer.py` and `ConfigurableTransformers`
set the order in which the transformers should apply
with the `pre_transformer_order` trait of this class
transformers registerd by this key will take precedence on
default one.
jinja_filters: dict of supplementary jinja filter that should be made
availlable in template. If those are of Configurable Class type,
they will be instanciated with the config object as argument.
user defined filter will overwrite the one availlable by default.
"""
super(ConverterTemplate, self).__init__(config=config, **kw)
# variable parameters depending on the pype of jinja environement
self.env = texenv if self.tex_environement else env
self.ext = '.tplx' if self.tex_environement else '.tpl'
for name in self.pre_transformer_order:
# get the user-defined transformer first
transformer = preprocessors.get(name, getattr(trans, name, None))
if isinstance(transformer, MetaHasTraits):
transformer = transformer(config=config)
self.preprocessors.append(transformer)
## for compat, remove later
self.preprocessors.append(trans.coalesce_streams)
self.preprocessors.append(
trans.ExtractFigureTransformer(config=config))
self.preprocessors.append(trans.RevealHelpTransformer(config=config))
self.preprocessors.append(
trans.CSSHtmlHeaderTransformer(config=config))
if SphinxTransformer:
self.preprocessors.append(SphinxTransformer(config=config))
self.preprocessors.append(LatexTransformer(config=config))
##
self.env.filters['filter_data_type'] = FilterDataType(config=config)
self.env.filters['pycomment'] = python_comment
self.env.filters['indent'] = indent
self.env.filters['rm_fake'] = rm_fake
self.env.filters['rm_ansi'] = remove_ansi
self.env.filters['markdown'] = markdown
self.env.filters['highlight'] = highlight
self.env.filters['highlight2latex'] = highlight2latex
self.env.filters['ansi2html'] = ansi2html
self.env.filters['markdown2latex'] = markdown2latex
self.env.filters['markdown2rst'] = markdown2rst
self.env.filters['get_lines'] = get_lines
self.env.filters['wrap'] = wrap
self.env.filters['rm_dollars'] = rm_dollars
## user filter will overwrite
for key, filtr in jinja_filters.iteritems():
if isinstance(filtr, MetaHasTraits):
self.env.filters[key] = filtr(config=config)
else:
self.env.filters[key] = filtr
self.template = self.env.get_template(self.template_file + self.ext)
def process(self, nb):
"""
preprocess the notebook json for easier use with the templates.
will call all the `preprocessor`s in order before returning it.
"""
# dict of 'resources' that could be made by the preprocessors
# like key/value data to extract files from ipynb like in latex conversion
resources = {}
for preprocessor in self.preprocessors:
nb, resources = preprocessor(nb, resources)
return nb, resources
def convert(self, nb):
""" convert the ipynb file
return both the converted ipynb file and a dict containing potential
other resources
"""
nb, resources = self.process(nb)
return self.template.render(nb=nb, resources=resources), resources
def from_filename(self, filename):
"""read and convert a notebook from a file name"""
with io.open(filename) as f:
return self.convert(nbformat.read(f, 'json'))
def from_file(self, filelike):
"""read and convert a notebook from a filelike object
filelike object will just be "read" and should be json format..
"""
return self.convert(nbformat.read(filelike, 'json'))
def from_json(self, json):
""" not implemented
Should convert from a json object
"""
raise NotImplementedError('not implemented (yet?)')
class InlineBackend(InlineBackendConfig):
"""An object to store configuration of the inline backend."""
def _config_changed(self, name, old, new):
# warn on change of renamed config section
if new.InlineBackendConfig != getattr(old, 'InlineBackendConfig', Config()):
warn("InlineBackendConfig has been renamed to InlineBackend")
super(InlineBackend, self)._config_changed(name, old, new)
# The typical default figure size is too large for inline use,
# so we shrink the figure size to 6x4, and tweak fonts to
# make that fit.
rc = Dict({'figure.figsize': (6.0,4.0),
# play nicely with white background in the Qt and notebook frontend
'figure.facecolor': (1,1,1,0),
'figure.edgecolor': (1,1,1,0),
'axes.facecolor': (1,1,1,0),
# 12pt labels get cutoff on 6x4 logplots, so use 10pt.
'font.size': 10,
# 72 dpi matches SVG/qtconsole
# this only affects PNG export, as SVG has no dpi setting
'savefig.dpi': 72,
# 10pt still needs a little more room on the xlabel:
'figure.subplot.bottom' : .125
}, config=True,
help="""Subset of matplotlib rcParams that should be different for the
inline backend."""
)
figure_formats = Set({'png'}, config=True,
help="""A set of figure formats to enable: 'png',
'retina', 'jpeg', 'svg', 'pdf'.""")
def _update_figure_formatters(self):
if self.shell is not None:
from IPython.core.pylabtools import select_figure_formats
select_figure_formats(self.shell, self.figure_formats, **self.print_figure_kwargs)
def _figure_formats_changed(self, name, old, new):
if 'jpg' in new or 'jpeg' in new:
if not pil_available():
raise TraitError("Requires PIL/Pillow for JPG figures")
self._update_figure_formatters()
figure_format = Unicode(config=True, help="""The figure format to enable (deprecated
use `figure_formats` instead)""")
def _figure_format_changed(self, name, old, new):
if new:
self.figure_formats = {new}
print_figure_kwargs = Dict({'bbox_inches' : 'tight'}, config=True,
help="""Extra kwargs to be passed to fig.canvas.print_figure.
Logical examples include: bbox_inches, quality (for jpeg figures), etc.
"""
)
_print_figure_kwargs_changed = _update_figure_formatters
close_figures = Bool(True, config=True,
help="""Close all figures at the end of each cell.
When True, ensures that each cell starts with no active figures, but it
also means that one must keep track of references in order to edit or
redraw figures in subsequent cells. This mode is ideal for the notebook,
where residual plots from other cells might be surprising.
When False, one must call figure() to create new figures. This means
that gcf() and getfigs() can reference figures created in other cells,
and the active figure can continue to be edited with pylab/pyplot
methods that reference the current active figure. This mode facilitates
iterative editing of figures, and behaves most consistently with
other matplotlib backends, but figure barriers between cells must
be explicit.
""")
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
class JupyterHub(Application):
"""An Application for starting a Multi-User Jupyter Notebook server."""
name = 'jupyterhub'
description = """Start a multi-user Jupyter Notebook server
Spawns a configurable-http-proxy and multi-user Hub,
which authenticates users and spawns single-user Notebook servers
on behalf of users.
"""
examples = """
generate default config file:
jupyterhub --generate-config -f /etc/jupyterhub/jupyterhub.py
spawn the server on 10.0.1.2:443 with https:
jupyterhub --ip 10.0.1.2 --port 443 --ssl-key my_ssl.key --ssl-cert my_ssl.cert
"""
aliases = Dict(aliases)
flags = Dict(flags)
subcommands = {'token': (NewToken, "Generate an API token for a user")}
classes = List([
Spawner,
LocalProcessSpawner,
Authenticator,
PAMAuthenticator,
])
config_file = Unicode(
'jupyterhub_config.py',
config=True,
help="The config file to load",
)
generate_config = Bool(
False,
config=True,
help="Generate default config file",
)
answer_yes = Bool(
False,
config=True,
help="Answer yes to any questions (e.g. confirm overwrite)")
pid_file = Unicode('',
config=True,
help="""File to write PID
Useful for daemonizing jupyterhub.
""")
last_activity_interval = Integer(
300,
config=True,
help=
"Interval (in seconds) at which to update last-activity timestamps.")
proxy_check_interval = Integer(
30,
config=True,
help="Interval (in seconds) at which to check if the proxy is running."
)
data_files_path = Unicode(
DATA_FILES_PATH,
config=True,
help=
"The location of jupyterhub data files (e.g. /usr/local/share/jupyter/hub)"
)
ssl_key = Unicode(
'',
config=True,
help="""Path to SSL key file for the public facing interface of the proxy
Use with ssl_cert
""")
ssl_cert = Unicode(
'',
config=True,
help=
"""Path to SSL certificate file for the public facing interface of the proxy
Use with ssl_key
""")
ip = Unicode('', config=True, help="The public facing ip of the proxy")
port = Integer(8000,
config=True,
help="The public facing port of the proxy")
base_url = URLPrefix('/',
config=True,
help="The base URL of the entire application")
jinja_environment_options = Dict(
config=True,
help="Supply extra arguments that will be passed to Jinja environment."
)
proxy_cmd = Unicode('configurable-http-proxy',
config=True,
help="""The command to start the http proxy.
Only override if configurable-http-proxy is not on your PATH
""")
debug_proxy = Bool(False,
config=True,
help="show debug output in configurable-http-proxy")
proxy_auth_token = Unicode(config=True,
help="""The Proxy Auth token.
Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default.
""")
def _proxy_auth_token_default(self):
token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None)
if not token:
self.log.warn('\n'.join([
"",
"Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.",
"Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.",
"",
]))
token = orm.new_token()
return token
proxy_api_ip = Unicode('localhost',
config=True,
help="The ip for the proxy API handlers")
proxy_api_port = Integer(config=True,
help="The port for the proxy API handlers")
def _proxy_api_port_default(self):
return self.port + 1
hub_port = Integer(8081, config=True, help="The port for this process")
hub_ip = Unicode('localhost', config=True, help="The ip for this process")
hub_prefix = URLPrefix(
'/hub/',
config=True,
help="The prefix for the hub server. Must not be '/'")
def _hub_prefix_default(self):
return url_path_join(self.base_url, '/hub/')
def _hub_prefix_changed(self, name, old, new):
if new == '/':
raise TraitError("'/' is not a valid hub prefix")
if not new.startswith(self.base_url):
self.hub_prefix = url_path_join(self.base_url, new)
cookie_secret = Bytes(config=True,
env='JPY_COOKIE_SECRET',
help="""The cookie secret to use to encrypt cookies.
Loaded from the JPY_COOKIE_SECRET env variable by default.
""")
cookie_secret_file = Unicode(
'jupyterhub_cookie_secret',
config=True,
help="""File in which to store the cookie secret.""")
authenticator_class = Type(PAMAuthenticator,
Authenticator,
config=True,
help="""Class for authenticating users.
This should be a class with the following form:
- constructor takes one kwarg: `config`, the IPython config object.
- is a tornado.gen.coroutine
- returns username on success, None on failure
- takes two arguments: (handler, data),
where `handler` is the calling web.RequestHandler,
and `data` is the POST form data from the login page.
""")
authenticator = Instance(Authenticator)
def _authenticator_default(self):
return self.authenticator_class(parent=self, db=self.db)
# class for spawning single-user servers
spawner_class = Type(
LocalProcessSpawner,
Spawner,
config=True,
help="""The class to use for spawning single-user servers.
Should be a subclass of Spawner.
""")
db_url = Unicode(
'sqlite:///jupyterhub.sqlite',
config=True,
help="url for the database. e.g. `sqlite:///jupyterhub.sqlite`")
def _db_url_changed(self, name, old, new):
if '://' not in new:
# assume sqlite, if given as a plain filename
self.db_url = 'sqlite:///%s' % new
db_kwargs = Dict(
config=True,
help="""Include any kwargs to pass to the database connection.
See sqlalchemy.create_engine for details.
""")
reset_db = Bool(False, config=True, help="Purge and reset the database.")
debug_db = Bool(
False,
config=True,
help="log all database transactions. This has A LOT of output")
db = Any()
session_factory = Any()
admin_access = Bool(
False,
config=True,
help="""Grant admin users permission to access single-user servers.
Users should be properly informed if this is enabled.
""")
admin_users = Set(config=True,
help="""set of usernames of admin users
If unspecified, only the user that launches the server will be admin.
""")
tornado_settings = Dict(config=True)
cleanup_servers = Bool(
True,
config=True,
help="""Whether to shutdown single-user servers when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the single-user servers running.
If both this and cleanup_proxy are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
""")
cleanup_proxy = Bool(
True,
config=True,
help="""Whether to shutdown the proxy when the Hub shuts down.
Disable if you want to be able to teardown the Hub while leaving the proxy running.
Only valid if the proxy was starting by the Hub process.
If both this and cleanup_servers are False, sending SIGINT to the Hub will
only shutdown the Hub, leaving everything else running.
The Hub should be able to resume from database state.
""")
handlers = List()
_log_formatter_cls = CoroutineLogFormatter
http_server = None
proxy_process = None
io_loop = None
def _log_level_default(self):
return logging.INFO
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%Y-%m-%d %H:%M:%S"
def _log_format_default(self):
"""override default log format to include time"""
return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s"
extra_log_file = Unicode("",
config=True,
help="Set a logging.FileHandler on this file.")
extra_log_handlers = List(
Instance(logging.Handler),
config=True,
help="Extra log handlers to set on JupyterHub logger",
)
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
if self.extra_log_file:
self.extra_log_handlers.append(
logging.FileHandler(self.extra_log_file))
_formatter = self._log_formatter_cls(
fmt=self.log_format,
datefmt=self.log_datefmt,
)
for handler in self.extra_log_handlers:
if handler.formatter is None:
handler.setFormatter(_formatter)
self.log.addHandler(handler)
# hook up tornado 3's loggers to our app handlers
for log in (app_log, access_log, gen_log):
# ensure all log statements identify the application they come from
log.name = self.log.name
logger = logging.getLogger('tornado')
logger.propagate = True
logger.parent = self.log
logger.setLevel(self.log.level)
def init_ports(self):
if self.hub_port == self.port:
raise TraitError(
"The hub and proxy cannot both listen on port %i" % self.port)
if self.hub_port == self.proxy_api_port:
raise TraitError(
"The hub and proxy API cannot both listen on port %i" %
self.hub_port)
if self.proxy_api_port == self.port:
raise TraitError(
"The proxy's public and API ports cannot both be %i" %
self.port)
@staticmethod
def add_url_prefix(prefix, handlers):
"""add a url prefix to handlers"""
for i, tup in enumerate(handlers):
lis = list(tup)
lis[0] = url_path_join(prefix, tup[0])
handlers[i] = tuple(lis)
return handlers
def init_handlers(self):
h = []
h.extend(handlers.default_handlers)
h.extend(apihandlers.default_handlers)
# load handlers from the authenticator
h.extend(self.authenticator.get_handlers(self))
self.handlers = self.add_url_prefix(self.hub_prefix, h)
# some extra handlers, outside hub_prefix
self.handlers.extend([
(r"%s" % self.hub_prefix.rstrip('/'), web.RedirectHandler, {
"url": self.hub_prefix,
"permanent": False,
}),
(r"(?!%s).*" % self.hub_prefix, handlers.PrefixRedirectHandler),
(r'(.*)', handlers.Template404),
])
def _check_db_path(self, path):
"""More informative log messages for failed filesystem access"""
path = os.path.abspath(path)
parent, fname = os.path.split(path)
user = getuser()
if not os.path.isdir(parent):
self.log.error("Directory %s does not exist", parent)
if os.path.exists(parent) and not os.access(parent, os.W_OK):
self.log.error("%s cannot create files in %s", user, parent)
if os.path.exists(path) and not os.access(path, os.W_OK):
self.log.error("%s cannot edit %s", user, path)
def init_secrets(self):
trait_name = 'cookie_secret'
trait = self.traits()[trait_name]
env_name = trait.get_metadata('env')
secret_file = os.path.abspath(
os.path.expanduser(self.cookie_secret_file))
secret = self.cookie_secret
secret_from = 'config'
# load priority: 1. config, 2. env, 3. file
if not secret and os.environ.get(env_name):
secret_from = 'env'
self.log.info("Loading %s from env[%s]", trait_name, env_name)
secret = binascii.a2b_hex(os.environ[env_name])
if not secret and os.path.exists(secret_file):
secret_from = 'file'
perm = os.stat(secret_file).st_mode
if perm & 0o077:
self.log.error("Bad permissions on %s", secret_file)
else:
self.log.info("Loading %s from %s", trait_name, secret_file)
with open(secret_file) as f:
b64_secret = f.read()
try:
secret = binascii.a2b_base64(b64_secret)
except Exception as e:
self.log.error("%s does not contain b64 key: %s",
secret_file, e)
if not secret:
secret_from = 'new'
self.log.debug("Generating new %s", trait_name)
secret = os.urandom(SECRET_BYTES)
if secret_file and secret_from == 'new':
# if we generated a new secret, store it in the secret_file
self.log.info("Writing %s to %s", trait_name, secret_file)
b64_secret = binascii.b2a_base64(secret).decode('ascii')
with open(secret_file, 'w') as f:
f.write(b64_secret)
try:
os.chmod(secret_file, 0o600)
except OSError:
self.log.warn("Failed to set permissions on %s", secret_file)
# store the loaded trait value
self.cookie_secret = secret
def init_db(self):
"""Create the database connection"""
self.log.debug("Connecting to db: %s", self.db_url)
try:
self.session_factory = orm.new_session_factory(self.db_url,
reset=self.reset_db,
echo=self.debug_db,
**self.db_kwargs)
self.db = scoped_session(self.session_factory)()
except OperationalError as e:
self.log.error("Failed to connect to db: %s", self.db_url)
self.log.debug("Database error was:", exc_info=True)
if self.db_url.startswith('sqlite:///'):
self._check_db_path(self.db_url.split(':///', 1)[1])
self.exit(1)
def init_hub(self):
"""Load the Hub config into the database"""
self.hub = self.db.query(orm.Hub).first()
if self.hub is None:
self.hub = orm.Hub(server=orm.Server(
ip=self.hub_ip,
port=self.hub_port,
base_url=self.hub_prefix,
cookie_name='jupyter-hub-token',
))
self.db.add(self.hub)
else:
server = self.hub.server
server.ip = self.hub_ip
server.port = self.hub_port
server.base_url = self.hub_prefix
self.db.commit()
@gen.coroutine
def init_users(self):
"""Load users into and from the database"""
db = self.db
if not self.admin_users:
# add current user as admin if there aren't any others
admins = db.query(orm.User).filter(orm.User.admin == True)
if admins.first() is None:
self.admin_users.add(getuser())
new_users = []
for name in self.admin_users:
# ensure anyone specified as admin in config is admin in db
user = orm.User.find(db, name)
if user is None:
user = orm.User(name=name, admin=True)
new_users.append(user)
db.add(user)
else:
user.admin = True
# the admin_users config variable will never be used after this point.
# only the database values will be referenced.
whitelist = self.authenticator.whitelist
if not whitelist:
self.log.info(
"Not using whitelist. Any authenticated user will be allowed.")
# add whitelisted users to the db
for name in whitelist:
user = orm.User.find(db, name)
if user is None:
user = orm.User(name=name)
new_users.append(user)
db.add(user)
if whitelist:
# fill the whitelist with any users loaded from the db,
# so we are consistent in both directions.
# This lets whitelist be used to set up initial list,
# but changes to the whitelist can occur in the database,
# and persist across sessions.
for user in db.query(orm.User):
whitelist.add(user.name)
# The whitelist set and the users in the db are now the same.
# From this point on, any user changes should be done simultaneously
# to the whitelist set and user db, unless the whitelist is empty (all users allowed).
db.commit()
for user in new_users:
yield gen.maybe_future(self.authenticator.add_user(user))
db.commit()
user_summaries = ['']
def _user_summary(user):
parts = ['{0: >8}'.format(user.name)]
if user.admin:
parts.append('admin')
if user.server:
parts.append('running at %s' % user.server)
return ' '.join(parts)
@gen.coroutine
def user_stopped(user):
status = yield user.spawner.poll()
self.log.warn(
"User %s server stopped with exit code: %s",
user.name,
status,
)
yield self.proxy.delete_user(user)
yield user.stop()
for user in db.query(orm.User):
if not user.state:
# without spawner state, server isn't valid
user.server = None
user_summaries.append(_user_summary(user))
continue
self.log.debug("Loading state for %s from db", user.name)
user.spawner = spawner = self.spawner_class(
user=user,
hub=self.hub,
config=self.config,
db=self.db,
)
status = yield spawner.poll()
if status is None:
self.log.info("%s still running", user.name)
spawner.add_poll_callback(user_stopped, user)
spawner.start_polling()
else:
# user not running. This is expected if server is None,
# but indicates the user's server died while the Hub wasn't running
# if user.server is defined.
log = self.log.warn if user.server else self.log.debug
log("%s not running.", user.name)
user.server = None
user_summaries.append(_user_summary(user))
self.log.debug("Loaded users: %s", '\n'.join(user_summaries))
db.commit()
def init_proxy(self):
"""Load the Proxy config into the database"""
self.proxy = self.db.query(orm.Proxy).first()
if self.proxy is None:
self.proxy = orm.Proxy(
public_server=orm.Server(),
api_server=orm.Server(),
)
self.db.add(self.proxy)
self.db.commit()
self.proxy.auth_token = self.proxy_auth_token # not persisted
self.proxy.log = self.log
self.proxy.public_server.ip = self.ip
self.proxy.public_server.port = self.port
self.proxy.api_server.ip = self.proxy_api_ip
self.proxy.api_server.port = self.proxy_api_port
self.proxy.api_server.base_url = '/api/routes/'
self.db.commit()
@gen.coroutine
def start_proxy(self):
"""Actually start the configurable-http-proxy"""
# check for proxy
if self.proxy.public_server.is_up() or self.proxy.api_server.is_up():
# check for *authenticated* access to the proxy (auth token can change)
try:
yield self.proxy.get_routes()
except (HTTPError, OSError, socket.error) as e:
if isinstance(e, HTTPError) and e.code == 403:
msg = "Did CONFIGPROXY_AUTH_TOKEN change?"
else:
msg = "Is something else using %s?" % self.proxy.public_server.url
self.log.error(
"Proxy appears to be running at %s, but I can't access it (%s)\n%s",
self.proxy.public_server.url, e, msg)
self.exit(1)
return
else:
self.log.info("Proxy already running at: %s",
self.proxy.public_server.url)
self.proxy_process = None
return
env = os.environ.copy()
env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token
cmd = [
self.proxy_cmd,
'--ip',
self.proxy.public_server.ip,
'--port',
str(self.proxy.public_server.port),
'--api-ip',
self.proxy.api_server.ip,
'--api-port',
str(self.proxy.api_server.port),
'--default-target',
self.hub.server.host,
]
if self.debug_proxy:
cmd.extend(['--log-level', 'debug'])
if self.ssl_key:
cmd.extend(['--ssl-key', self.ssl_key])
if self.ssl_cert:
cmd.extend(['--ssl-cert', self.ssl_cert])
self.log.info("Starting proxy @ %s", self.proxy.public_server.url)
self.log.debug("Proxy cmd: %s", cmd)
self.proxy_process = Popen(cmd, env=env)
def _check():
status = self.proxy_process.poll()
if status is not None:
e = RuntimeError("Proxy failed to start with exit code %i" %
status)
# py2-compatible `raise e from None`
e.__cause__ = None
raise e
for server in (self.proxy.public_server, self.proxy.api_server):
for i in range(10):
_check()
try:
yield server.wait_up(1)
except TimeoutError:
continue
else:
break
yield server.wait_up(1)
self.log.debug("Proxy started and appears to be up")
@gen.coroutine
def check_proxy(self):
if self.proxy_process.poll() is None:
return
self.log.error(
"Proxy stopped with exit code %r", 'unknown'
if self.proxy_process is None else self.proxy_process.poll())
yield self.start_proxy()
self.log.info("Setting up routes on new proxy")
yield self.proxy.add_all_users()
self.log.info("New proxy back up, and good to go")
def init_tornado_settings(self):
"""Set up the tornado settings dict."""
base_url = self.hub.server.base_url
template_path = os.path.join(self.data_files_path, 'templates'),
jinja_env = Environment(loader=FileSystemLoader(template_path),
**self.jinja_environment_options)
login_url = self.authenticator.login_url(base_url)
logout_url = self.authenticator.logout_url(base_url)
# if running from git, disable caching of require.js
# otherwise cache based on server start time
parent = os.path.dirname(os.path.dirname(jupyterhub.__file__))
if os.path.isdir(os.path.join(parent, '.git')):
version_hash = ''
else:
version_hash = datetime.now().strftime("%Y%m%d%H%M%S"),
settings = dict(
config=self.config,
log=self.log,
db=self.db,
proxy=self.proxy,
hub=self.hub,
admin_users=self.admin_users,
admin_access=self.admin_access,
authenticator=self.authenticator,
spawner_class=self.spawner_class,
base_url=self.base_url,
cookie_secret=self.cookie_secret,
login_url=login_url,
logout_url=logout_url,
static_path=os.path.join(self.data_files_path, 'static'),
static_url_prefix=url_path_join(self.hub.server.base_url,
'static/'),
static_handler_class=CacheControlStaticFilesHandler,
template_path=template_path,
jinja2_env=jinja_env,
version_hash=version_hash,
)
# allow configured settings to have priority
settings.update(self.tornado_settings)
self.tornado_settings = settings
def init_tornado_application(self):
"""Instantiate the tornado Application object"""
self.tornado_application = web.Application(self.handlers,
**self.tornado_settings)
def write_pid_file(self):
pid = os.getpid()
if self.pid_file:
self.log.debug("Writing PID %i to %s", pid, self.pid_file)
with open(self.pid_file, 'w') as f:
f.write('%i' % pid)
@gen.coroutine
@catch_config_error
def initialize(self, *args, **kwargs):
super().initialize(*args, **kwargs)
if self.generate_config or self.subapp:
return
self.load_config_file(self.config_file)
self.init_logging()
if 'JupyterHubApp' in self.config:
self.log.warn(
"Use JupyterHub in config, not JupyterHubApp. Outdated config:\n%s",
'\n'.join('JupyterHubApp.{key} = {value!r}'.format(key=key,
value=value)
for key, value in self.config.JupyterHubApp.items()))
cfg = self.config.copy()
cfg.JupyterHub.merge(cfg.JupyterHubApp)
self.update_config(cfg)
self.write_pid_file()
self.init_ports()
self.init_secrets()
self.init_db()
self.init_hub()
self.init_proxy()
yield self.init_users()
self.init_handlers()
self.init_tornado_settings()
self.init_tornado_application()
@gen.coroutine
def cleanup(self):
"""Shutdown our various subprocesses and cleanup runtime files."""
futures = []
if self.cleanup_servers:
self.log.info("Cleaning up single-user servers...")
# request (async) process termination
for user in self.db.query(orm.User):
if user.spawner is not None:
futures.append(user.stop())
else:
self.log.info("Leaving single-user servers running")
# clean up proxy while SUS are shutting down
if self.cleanup_proxy:
if self.proxy_process:
self.log.info("Cleaning up proxy[%i]...",
self.proxy_process.pid)
if self.proxy_process.poll() is None:
try:
self.proxy_process.terminate()
except Exception as e:
self.log.error("Failed to terminate proxy process: %s",
e)
else:
self.log.info("I didn't start the proxy, I can't clean it up")
else:
self.log.info("Leaving proxy running")
# wait for the requests to stop finish:
for f in futures:
try:
yield f
except Exception as e:
self.log.error("Failed to stop user: %s", e)
self.db.commit()
if self.pid_file and os.path.exists(self.pid_file):
self.log.info("Cleaning up PID file %s", self.pid_file)
os.remove(self.pid_file)
# finally stop the loop once we are all cleaned up
self.log.info("...done")
def write_config_file(self):
"""Write our default config to a .py config file"""
if os.path.exists(self.config_file) and not self.answer_yes:
answer = ''
def ask():
prompt = "Overwrite %s with default config? [y/N]" % self.config_file
try:
return input(prompt).lower() or 'n'
except KeyboardInterrupt:
print('') # empty line
return 'n'
answer = ask()
while not answer.startswith(('y', 'n')):
print("Please answer 'yes' or 'no'")
answer = ask()
if answer.startswith('n'):
return
config_text = self.generate_config_file()
if isinstance(config_text, bytes):
config_text = config_text.decode('utf8')
print("Writing default config to: %s" % self.config_file)
with open(self.config_file, mode='w') as f:
f.write(config_text)
@gen.coroutine
def update_last_activity(self):
"""Update User.last_activity timestamps from the proxy"""
routes = yield self.proxy.get_routes()
for prefix, route in routes.items():
if 'user' not in route:
# not a user route, ignore it
continue
user = orm.User.find(self.db, route['user'])
if user is None:
self.log.warn("Found no user for route: %s", route)
continue
try:
dt = datetime.strptime(route['last_activity'], ISO8601_ms)
except Exception:
dt = datetime.strptime(route['last_activity'], ISO8601_s)
user.last_activity = max(user.last_activity, dt)
self.db.commit()
yield self.proxy.check_routes(routes)
@gen.coroutine
def start(self):
"""Start the whole thing"""
self.io_loop = loop = IOLoop.current()
if self.subapp:
self.subapp.start()
loop.stop()
return
if self.generate_config:
self.write_config_file()
loop.stop()
return
# start the proxy
try:
yield self.start_proxy()
except Exception as e:
self.log.critical("Failed to start proxy", exc_info=True)
self.exit(1)
return
loop.add_callback(self.proxy.add_all_users)
if self.proxy_process:
# only check / restart the proxy if we started it in the first place.
# this means a restarted Hub cannot restart a Proxy that its
# predecessor started.
pc = PeriodicCallback(self.check_proxy,
1e3 * self.proxy_check_interval)
pc.start()
if self.last_activity_interval:
pc = PeriodicCallback(self.update_last_activity,
1e3 * self.last_activity_interval)
pc.start()
# start the webserver
self.http_server = tornado.httpserver.HTTPServer(
self.tornado_application, xheaders=True)
self.http_server.listen(self.hub_port)
# register cleanup on both TERM and INT
atexit.register(self.atexit)
signal.signal(signal.SIGTERM, self.sigterm)
def sigterm(self, signum, frame):
self.log.critical("Received SIGTERM, shutting down")
self.io_loop.stop()
self.atexit()
_atexit_ran = False
def atexit(self):
"""atexit callback"""
if self._atexit_ran:
return
self._atexit_ran = True
# run the cleanup step (in a new loop, because the interrupted one is unclean)
IOLoop.clear_current()
loop = IOLoop()
loop.make_current()
loop.run_sync(self.cleanup)
def stop(self):
if not self.io_loop:
return
if self.http_server:
self.io_loop.add_callback(self.http_server.stop)
self.io_loop.add_callback(self.io_loop.stop)
@gen.coroutine
def launch_instance_async(self, argv=None):
yield self.initialize(argv)
yield self.start()
@classmethod
def launch_instance(cls, argv=None):
self = cls.instance(argv=argv)
loop = IOLoop.current()
loop.add_callback(self.launch_instance_async, argv)
try:
loop.start()
except KeyboardInterrupt:
print("\nInterrupted")
class InteractiveShellApp(Configurable):
"""A Mixin for applications that start InteractiveShell instances.
Provides configurables for loading extensions and executing files
as part of configuring a Shell environment.
The following methods should be called by the :meth:`initialize` method
of the subclass:
- :meth:`init_path`
- :meth:`init_shell` (to be implemented by the subclass)
- :meth:`init_gui_pylab`
- :meth:`init_extensions`
- :meth:`init_code`
"""
extensions = List(
Unicode,
config=True,
help="A list of dotted module names of IPython extensions to load.")
extra_extension = Unicode(
'',
config=True,
help="dotted module name of an IPython extension to load.")
def _extra_extension_changed(self, name, old, new):
if new:
# add to self.extensions
self.extensions.append(new)
# Extensions that are always loaded (not configurable)
default_extensions = List(Unicode, [u'storemagic'], config=False)
hide_initial_ns = Bool(
True,
config=True,
help=
"""Should variables loaded at startup (by startup files, exec_lines, etc.)
be hidden from tools like %who?""")
exec_files = List(Unicode,
config=True,
help="""List of files to run at IPython startup.""")
file_to_run = Unicode('', config=True, help="""A file to be run""")
exec_lines = List(Unicode,
config=True,
help="""lines of code to run at IPython startup.""")
code_to_run = Unicode('',
config=True,
help="Execute the given command string.")
module_to_run = Unicode('',
config=True,
help="Run the module as a script.")
gui = CaselessStrEnum(
gui_keys,
config=True,
help="Enable GUI event loop integration with any of {0}.".format(
gui_keys))
matplotlib = CaselessStrEnum(
backend_keys,
config=True,
help="""Configure matplotlib for interactive use with
the default matplotlib backend.""")
pylab = CaselessStrEnum(
backend_keys,
config=True,
help="""Pre-load matplotlib and numpy for interactive use,
selecting a particular matplotlib backend and loop integration.
""")
pylab_import_all = Bool(
True,
config=True,
help=
"""If true, IPython will populate the user namespace with numpy, pylab, etc.
and an ``import *`` is done from numpy and pylab, when using pylab mode.
When False, pylab mode should not import any names into the user namespace.
""")
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
user_ns = Instance(dict, args=None, allow_none=True)
def _user_ns_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_ns = new
self.shell.init_user_ns()
def init_path(self):
"""Add current working directory, '', to sys.path"""
if sys.path[0] != '':
sys.path.insert(0, '')
def init_shell(self):
raise NotImplementedError("Override in subclasses")
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
enable = False
shell = self.shell
if self.pylab:
enable = lambda key: shell.enable_pylab(
key, import_all=self.pylab_import_all)
key = self.pylab
elif self.matplotlib:
enable = shell.enable_matplotlib
key = self.matplotlib
elif self.gui:
enable = shell.enable_gui
key = self.gui
if not enable:
return
try:
r = enable(key)
except ImportError:
self.log.warn(
"Eventloop or matplotlib integration failed. Is matplotlib installed?"
)
self.shell.showtraceback()
return
except Exception:
self.log.warn("GUI event loop or pylab initialization failed")
self.shell.showtraceback()
return
if isinstance(r, tuple):
gui, backend = r[:2]
self.log.info(
"Enabling GUI event loop integration, "
"eventloop=%s, matplotlib=%s", gui, backend)
if key == "auto":
print("Using matplotlib backend: %s" % backend)
else:
gui = r
self.log.info(
"Enabling GUI event loop integration, "
"eventloop=%s", gui)
def init_extensions(self):
"""Load all IPython extensions in IPythonApp.extensions.
This uses the :meth:`ExtensionManager.load_extensions` to load all
the extensions listed in ``self.extensions``.
"""
try:
self.log.debug("Loading IPython extensions...")
extensions = self.default_extensions + self.extensions
for ext in extensions:
try:
self.log.info("Loading IPython extension: %s" % ext)
self.shell.extension_manager.load_extension(ext)
except:
self.log.warn("Error in loading extension: %s" % ext +
"\nCheck your config files in %s" %
self.profile_dir.location)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in loading extensions:")
self.shell.showtraceback()
def init_code(self):
"""run the pre-flight code, specified via exec_lines"""
self._run_startup_files()
self._run_exec_lines()
self._run_exec_files()
# Hide variables defined here from %who etc.
if self.hide_initial_ns:
self.shell.user_ns_hidden.update(self.shell.user_ns)
# command-line execution (ipython -i script.py, ipython -m module)
# should *not* be excluded from %whos
self._run_cmd_line_code()
self._run_module()
# flush output, so itwon't be attached to the first cell
sys.stdout.flush()
sys.stderr.flush()
def _run_exec_lines(self):
"""Run lines of code in IPythonApp.exec_lines in the user's namespace."""
if not self.exec_lines:
return
try:
self.log.debug("Running code from IPythonApp.exec_lines...")
for line in self.exec_lines:
try:
self.log.info("Running code in user namespace: %s" % line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user "
"namespace: %s" % line)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
self.shell.showtraceback()
def _exec_file(self, fname):
try:
full_filename = filefind(fname, [u'.', self.ipython_dir])
except IOError as e:
self.log.warn("File not found: %r" % fname)
return
# Make sure that the running script gets a proper sys.argv as if it
# were run from a system shell.
save_argv = sys.argv
sys.argv = [full_filename] + self.extra_args[1:]
# protect sys.argv from potential unicode strings on Python 2:
if not py3compat.PY3:
sys.argv = [py3compat.cast_bytes(a) for a in sys.argv]
try:
if os.path.isfile(full_filename):
self.log.info("Running file in user namespace: %s" %
full_filename)
# Ensure that __file__ is always defined to match Python
# behavior.
with preserve_keys(self.shell.user_ns, '__file__'):
self.shell.user_ns['__file__'] = fname
if full_filename.endswith('.ipy'):
self.shell.safe_execfile_ipy(full_filename)
else:
# default to python, even without extension
self.shell.safe_execfile(full_filename,
self.shell.user_ns)
finally:
sys.argv = save_argv
def _run_startup_files(self):
"""Run files from profile startup directory"""
startup_dir = self.profile_dir.startup_dir
startup_files = []
if os.environ.get('PYTHONSTARTUP', False):
startup_files.append(os.environ['PYTHONSTARTUP'])
startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
if not startup_files:
return
self.log.debug("Running startup files from %s...", startup_dir)
try:
for fname in sorted(startup_files):
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling startup files:")
self.shell.showtraceback()
def _run_exec_files(self):
"""Run files from IPythonApp.exec_files"""
if not self.exec_files:
return
self.log.debug("Running files in IPythonApp.exec_files...")
try:
for fname in self.exec_files:
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling IPythonApp.exec_files:")
self.shell.showtraceback()
def _run_cmd_line_code(self):
"""Run code or file specified at the command-line"""
if self.code_to_run:
line = self.code_to_run
try:
self.log.info("Running code given at command line (c=): %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user namespace: %s" %
line)
self.shell.showtraceback()
# Like Python itself, ignore the second if the first of these is present
elif self.file_to_run:
fname = self.file_to_run
try:
self._exec_file(fname)
except:
self.log.warn("Error in executing file in user namespace: %s" %
fname)
self.shell.showtraceback()
def _run_module(self):
"""Run module specified at the command-line."""
if self.module_to_run:
# Make sure that the module gets a proper sys.argv as if it were
# run using `python -m`.
save_argv = sys.argv
sys.argv = [sys.executable] + self.extra_args
try:
self.shell.safe_run_module(self.module_to_run,
self.shell.user_ns)
finally:
sys.argv = save_argv
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager,
FileNotebookManager]
flags = Dict(flags)
aliases = Dict(aliases)
subcommands = dict(
list=(NbserverListApp, NbserverListApp.description.splitlines()[0]),
)
kernel_argv = List(Unicode)
def _log_level_default(self):
return logging.INFO
def _log_format_default(self):
"""override default log format to include time"""
return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s"
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('', config=True)
def _file_to_run_changed(self, name, old, new):
path, base = os.path.split(new)
if path:
self.file_to_run = base
self.notebook_dir = path
# Network related information.
ip = Unicode(config=True,
help="The IP address the notebook server will listen on."
)
def _ip_default(self):
return localhost()
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888, config=True,
help="The port the notebook server will listen on."
)
port_retries = Integer(50, config=True,
help="The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(u'', config=True,
help="""The full path to an SSL/TLS certificate file."""
)
keyfile = Unicode(u'', config=True,
help="""The full path to a private key file for usage with SSL/TLS."""
)
cookie_secret = Bytes(b'', config=True,
help="""The random bytes used to secure cookies.
By default this is a new random number every time you start the Notebook.
Set it to a value in a config file to enable logins to persist across server sessions.
Note: Cookie secrets should be kept private, do not share config files with
cookie_secret stored in plaintext (you can read the value from a file).
"""
)
def _cookie_secret_default(self):
return os.urandom(1024)
password = Unicode(u'', config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
"""
)
open_browser = Bool(True, config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'', config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
webapp_settings = Dict(config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
jinja_environment_options = Dict(config=True,
help="Supply extra arguments that will be passed to Jinja environment.")
enable_mathjax = Bool(True, config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
"""
)
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_url = Unicode('/', config=True,
help='''The base URL for the notebook server.
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_url = '/'+new
elif not new.endswith('/'):
self.base_url = new+'/'
base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""")
def _base_project_url_changed(self, name, old, new):
self.log.warn("base_project_url is deprecated, use base_url")
self.base_url = new
extra_static_paths = List(Unicode, config=True,
help="""Extra paths to search for serving static files.
This allows adding javascript/css to be available from the notebook server machine,
or overriding individual files in the IPython"""
)
def _extra_static_paths_default(self):
return [os.path.join(self.profile_dir.location, 'static')]
@property
def static_file_path(self):
"""return extra paths + the default location"""
return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
nbextensions_path = List(Unicode, config=True,
help="""paths for Javascript extensions. By default, this is just IPYTHONDIR/nbextensions"""
)
def _nbextensions_path_default(self):
return [os.path.join(get_ipython_dir(), 'nbextensions')]
mathjax_url = Unicode("", config=True,
help="""The url for MathJax.js."""
)
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_url_prefix = self.webapp_settings.get("static_url_prefix",
url_path_join(self.base_url, "static")
)
# try local mathjax, either in nbextensions/mathjax or static/mathjax
for (url_prefix, search_path) in [
(url_path_join(self.base_url, "nbextensions"), self.nbextensions_path),
(static_url_prefix, self.static_file_path),
]:
self.log.debug("searching for local mathjax in %s", search_path)
try:
mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path)
except IOError:
continue
else:
url = url_path_join(url_prefix, u"mathjax/MathJax.js")
self.log.info("Serving local MathJax from %s at %s", mathjax, url)
return url
# no local mathjax, serve from CDN
if self.certfile:
# HTTPS: load from Rackspace CDN, because SSL certificate requires it
host = u"https://c328740.ssl.cf1.rackcdn.com"
else:
host = u"http://cdn.mathjax.org"
url = host + u"/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
notebook_manager_class = DottedObjectName('IPython.html.services.notebooks.filenbmanager.FileNotebookManager',
config=True,
help='The notebook manager class to use.')
trust_xheaders = Bool(False, config=True,
help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
"sent by the upstream reverse proxy. Necessary if the proxy handles SSL")
)
info_file = Unicode()
def _info_file_default(self):
info_file = "nbserver-%s.json"%os.getpid()
return os.path.join(self.profile_dir.security_dir, info_file)
notebook_dir = Unicode(py3compat.getcwd(), config=True,
help="The directory to use for notebooks and kernels."
)
def _notebook_dir_changed(self, name, old, new):
"""Do a bit of validation of the notebook dir."""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
self.notebook_dir = os.path.abspath(new)
return
if not os.path.isdir(new):
raise TraitError("No such notebook dir: %r" % new)
# setting App.notebook_dir implies setting notebook and kernel dirs as well
self.config.FileNotebookManager.notebook_dir = new
self.config.MappingKernelManager.root_dir = new
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if self.extra_args:
arg0 = self.extra_args[0]
f = os.path.abspath(arg0)
self.argv.remove(arg0)
if not os.path.exists(f):
self.log.critical("No such file or directory: %s", f)
self.exit(1)
# Use config here, to ensure that it takes higher priority than
# anything that comes from the profile.
if os.path.isdir(f):
self.config.NotebookApp.notebook_dir = f
elif os.path.isfile(f):
self.config.NotebookApp.file_to_run = f
def init_kernel_argv(self):
"""construct the kernel arguments"""
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(self.argv, notebook_aliases, notebook_flags)
if any(arg.startswith(u'--pylab') for arg in self.kernel_argv):
self.log.warn('\n '.join([
"Starting all kernels in pylab mode is not recommended,",
"and will be disabled in a future release.",
"Please use the %matplotlib magic to enable matplotlib instead.",
"pylab implies many imports, which can have confusing side effects",
"and harm the reproducibility of your notebooks.",
]))
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name)
# Kernel should get *absolute* path to profile directory
self.kernel_argv.extend(["--profile-dir", self.profile_dir.location])
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_manager = MappingKernelManager(
parent=self, log=self.log, kernel_argv=self.kernel_argv,
connection_dir = self.profile_dir.security_dir,
)
kls = import_item(self.notebook_manager_class)
self.notebook_manager = kls(parent=self, log=self.log)
self.session_manager = SessionManager(parent=self, log=self.log)
self.cluster_manager = ClusterManager(parent=self, log=self.log)
self.cluster_manager.update_profiles()
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
# hook up tornado 3's loggers to our app handlers
for name in ('access', 'application', 'general'):
logger = logging.getLogger('tornado.%s' % name)
logger.parent = self.log
logger.setLevel(self.log.level)
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(
self, self.kernel_manager, self.notebook_manager,
self.cluster_manager, self.session_manager,
self.log, self.base_url, self.webapp_settings,
self.jinja_environment_options
)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options,
xheaders=self.trust_xheaders)
if not self.ip:
warning = "WARNING: The notebook server is listening on all IP addresses"
if ssl_options is None:
self.log.critical(warning + " and not using encryption. This "
"is not recommended.")
if not self.password:
self.log.critical(warning + " and not using authentication. "
"This is highly insecure and not recommended.")
success = None
for port in random_ports(self.port, self.port_retries+1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
if e.errno == errno.EADDRINUSE:
self.log.info('The port %i is already in use, trying another random port.' % port)
continue
elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)):
self.log.warn("Permission to listen on port %i denied" % port)
continue
else:
raise
else:
self.port = port
success = True
break
if not success:
self.log.critical('ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
@property
def display_url(self):
ip = self.ip if self.ip else '[all ip addresses on your system]'
return self._url(ip)
@property
def connection_url(self):
ip = self.ip if self.ip else localhost()
return self._url(ip)
def _url(self, ip):
proto = 'https' if self.certfile else 'http'
return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url)
def init_signal(self):
if not sys.platform.startswith('win'):
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
if hasattr(signal, 'SIGUSR1'):
# Windows doesn't support SIGUSR1
signal.signal(signal.SIGUSR1, self._signal_info)
if hasattr(signal, 'SIGINFO'):
# only on BSD-based systems
signal.signal(signal.SIGINFO, self._signal_info)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
# FIXME: remove this delay when pyzmq dependency is >= 2.1.11
time.sleep(0.1)
info = self.log.info
info('interrupted')
print(self.notebook_info())
sys.stdout.write("Shutdown this notebook server (y/[n])? ")
sys.stdout.flush()
r,w,x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y'):
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.instance().stop()
return
else:
print("No answer for 5s:", end=' ')
print("resuming operation...")
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.instance().stop()
def _signal_info(self, sig, frame):
print(self.notebook_info())
def init_components(self):
"""Check the components submodule, and warn if it's unclean"""
status = submodule.check_submodule_status()
if status == 'missing':
self.log.warn("components submodule missing, running `git submodule update`")
submodule.update_submodules(submodule.ipython_parent())
elif status == 'unclean':
self.log.warn("components submodule unclean, you may see 404s on static/components")
self.log.warn("run `setup.py submodule` or `git submodule update` to update")
@catch_config_error
def initialize(self, argv=None):
super(NotebookApp, self).initialize(argv)
self.init_logging()
self.init_kernel_argv()
self.init_configurables()
self.init_components()
self.init_webapp()
self.init_signal()
def cleanup_kernels(self):
"""Shutdown all kernels.
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
self.kernel_manager.shutdown_all()
def notebook_info(self):
"Return the current working directory and the server url information"
info = self.notebook_manager.info_string() + "\n"
info += "%d active kernels \n" % len(self.kernel_manager._kernels)
return info + "The IPython Notebook is running at: %s" % self.display_url
def server_info(self):
"""Return a JSONable dict of information about this server."""
return {'url': self.connection_url,
'hostname': self.ip if self.ip else 'localhost',
'port': self.port,
'secure': bool(self.certfile),
'base_url': self.base_url,
'notebook_dir': os.path.abspath(self.notebook_dir),
}
def write_server_info_file(self):
"""Write the result of server_info() to the JSON file info_file."""
with open(self.info_file, 'w') as f:
json.dump(self.server_info(), f, indent=2)
def remove_server_info_file(self):
"""Remove the nbserver-<pid>.json file created for this server.
Ignores the error raised when the file has already been removed.
"""
try:
os.unlink(self.info_file)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def start(self):
""" Start the IPython Notebook server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
if self.subapp is not None:
return self.subapp.start()
info = self.log.info
for line in self.notebook_info().split("\n"):
info(line)
info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")
self.write_server_info_file()
if self.open_browser or self.file_to_run:
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
fullpath = os.path.join(self.notebook_dir, self.file_to_run)
if not os.path.exists(fullpath):
self.log.critical("%s does not exist" % fullpath)
self.exit(1)
uri = url_path_join('notebooks', self.file_to_run)
else:
uri = 'tree'
if browser:
b = lambda : browser.open(url_path_join(self.connection_url, uri),
new=2)
threading.Thread(target=b).start()
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
self.remove_server_info_file()
class FileContentsManager(FileManagerMixin, ContentsManager):
root_dir = Unicode(config=True)
def _root_dir_default(self):
try:
return self.parent.notebook_dir
except AttributeError:
return getcwd()
save_script = Bool(False,
config=True,
help='DEPRECATED, use post_save_hook')
def _save_script_changed(self):
self.log.warn("""
`--script` is deprecated. You can trigger nbconvert via pre- or post-save hooks:
ContentsManager.pre_save_hook
FileContentsManager.post_save_hook
A post-save hook has been registered that calls:
ipython nbconvert --to script [notebook]
which behaves similarly to `--script`.
""")
self.post_save_hook = _post_save_script
post_save_hook = Any(None,
config=True,
help="""Python callable or importstring thereof
to be called on the path of a file just saved.
This can be used to process the file on disk,
such as converting the notebook to a script or HTML via nbconvert.
It will be called as (all arguments passed by keyword)::
hook(os_path=os_path, model=model, contents_manager=instance)
- path: the filesystem path to the file just written
- model: the model representing the file
- contents_manager: this ContentsManager instance
""")
def _post_save_hook_changed(self, name, old, new):
if new and isinstance(new, string_types):
self.post_save_hook = import_item(self.post_save_hook)
elif new:
if not callable(new):
raise TraitError("post_save_hook must be callable")
def run_post_save_hook(self, model, os_path):
"""Run the post-save hook if defined, and log errors"""
if self.post_save_hook:
try:
self.log.debug("Running post-save hook on %s", os_path)
self.post_save_hook(os_path=os_path,
model=model,
contents_manager=self)
except Exception:
self.log.error("Post-save hook failed on %s",
os_path,
exc_info=True)
def _root_dir_changed(self, name, old, new):
"""Do a bit of validation of the root_dir."""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
self.root_dir = os.path.abspath(new)
return
if not os.path.isdir(new):
raise TraitError("%r is not a directory" % new)
def _checkpoints_class_default(self):
return FileCheckpoints
def is_hidden(self, path):
"""Does the API style path correspond to a hidden directory or file?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
hidden : bool
Whether the path exists and is hidden.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return is_hidden(os_path, self.root_dir)
def file_exists(self, path):
"""Returns True if the file exists, else returns False.
API-style wrapper for os.path.isfile
Parameters
----------
path : string
The relative path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the file exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path)
return os.path.isfile(os_path)
def dir_exists(self, path):
"""Does the API-style path refer to an extant directory?
API-style wrapper for os.path.isdir
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to root_dir).
Returns
-------
exists : bool
Whether the path is indeed a directory.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.isdir(os_path)
def exists(self, path):
"""Returns True if the path exists, else returns False.
API-style wrapper for os.path.exists
Parameters
----------
path : string
The API path to the file (with '/' as separator)
Returns
-------
exists : bool
Whether the target exists.
"""
path = path.strip('/')
os_path = self._get_os_path(path=path)
return os.path.exists(os_path)
def _base_model(self, path):
"""Build the common base of a contents model"""
os_path = self._get_os_path(path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the base model.
model = {}
model['name'] = path.rsplit('/', 1)[-1]
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
model['content'] = None
model['format'] = None
model['mimetype'] = None
try:
model['writable'] = os.access(os_path, os.W_OK)
except OSError:
self.log.error("Failed to check write permissions on %s", os_path)
model['writable'] = False
return model
def _dir_model(self, path, content=True):
"""Build a model for a directory
if content is requested, will include a listing of the directory
"""
os_path = self._get_os_path(path)
four_o_four = u'directory does not exist: %r' % path
if not os.path.isdir(os_path):
raise web.HTTPError(404, four_o_four)
elif is_hidden(os_path, self.root_dir):
self.log.info(
"Refusing to serve hidden directory %r, via 404 Error",
os_path)
raise web.HTTPError(404, four_o_four)
model = self._base_model(path)
model['type'] = 'directory'
if content:
model['content'] = contents = []
os_dir = self._get_os_path(path)
for name in os.listdir(os_dir):
os_path = os.path.join(os_dir, name)
# skip over broken symlinks in listing
if not os.path.exists(os_path):
self.log.warn("%s doesn't exist", os_path)
continue
elif not os.path.isfile(os_path) and not os.path.isdir(
os_path):
self.log.debug("%s not a regular file", os_path)
continue
if self.should_list(name) and not is_hidden(
os_path, self.root_dir):
contents.append(
self.get(path='%s/%s' % (path, name), content=False))
model['format'] = 'json'
return model
def _file_model(self, path, content=True, format=None):
"""Build a model for a file
if content is requested, include the file contents.
format:
If 'text', the contents will be decoded as UTF-8.
If 'base64', the raw bytes contents will be encoded as base64.
If not specified, try to decode as UTF-8, and fall back to base64
"""
model = self._base_model(path)
model['type'] = 'file'
os_path = self._get_os_path(path)
model['mimetype'] = mimetypes.guess_type(os_path)[0]
if content:
content, format = self._read_file(os_path, format)
if model['mimetype'] is None:
default_mime = {
'text': 'text/plain',
'base64': 'application/octet-stream'
}[format]
model['mimetype'] = default_mime
model.update(
content=content,
format=format,
)
return model
def _notebook_model(self, path, content=True):
"""Build a notebook model
if content is requested, the notebook content will be populated
as a JSON structure (not double-serialized)
"""
model = self._base_model(path)
model['type'] = 'notebook'
if content:
os_path = self._get_os_path(path)
nb = self._read_notebook(os_path, as_version=4)
self.mark_trusted_cells(nb, path)
model['content'] = nb
model['format'] = 'json'
self.validate_notebook_model(model)
return model
def get(self, path, content=True, type=None, format=None):
""" Takes a path for an entity and returns its model
Parameters
----------
path : str
the API path that describes the relative path for the target
content : bool
Whether to include the contents in the reply
type : str, optional
The requested type - 'file', 'notebook', or 'directory'.
Will raise HTTPError 400 if the content doesn't match.
format : str, optional
The requested format for file contents. 'text' or 'base64'.
Ignored if this returns a notebook or directory model.
Returns
-------
model : dict
the contents model. If content=True, returns the contents
of the file or directory as well.
"""
path = path.strip('/')
if not self.exists(path):
raise web.HTTPError(404, u'No such file or directory: %s' % path)
os_path = self._get_os_path(path)
if os.path.isdir(os_path):
if type not in (None, 'directory'):
raise web.HTTPError(400,
u'%s is a directory, not a %s' %
(path, type),
reason='bad type')
model = self._dir_model(path, content=content)
elif type == 'notebook' or (type is None and path.endswith('.ipynb')):
model = self._notebook_model(path, content=content)
else:
if type == 'directory':
raise web.HTTPError(400,
u'%s is not a directory' % path,
reason='bad type')
model = self._file_model(path, content=content, format=format)
return model
def _save_directory(self, os_path, model, path=''):
"""create a directory"""
if is_hidden(os_path, self.root_dir):
raise web.HTTPError(400,
u'Cannot create hidden directory %r' % os_path)
if not os.path.exists(os_path):
with self.perm_to_403():
os.mkdir(os_path)
elif not os.path.isdir(os_path):
raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
else:
self.log.debug("Directory %r already exists", os_path)
def save(self, model, path=''):
"""Save the file model and return the model with no content."""
path = path.strip('/')
if 'type' not in model:
raise web.HTTPError(400, u'No file type provided')
if 'content' not in model and model['type'] != 'directory':
raise web.HTTPError(400, u'No file content provided')
os_path = self._get_os_path(path)
self.log.debug("Saving %s", os_path)
self.run_pre_save_hook(model=model, path=path)
try:
if model['type'] == 'notebook':
nb = nbformat.from_dict(model['content'])
self.check_and_sign(nb, path)
self._save_notebook(os_path, nb)
# One checkpoint should always exist for notebooks.
if not self.checkpoints.list_checkpoints(path):
self.create_checkpoint(path)
elif model['type'] == 'file':
# Missing format will be handled internally by _save_file.
self._save_file(os_path, model['content'], model.get('format'))
elif model['type'] == 'directory':
self._save_directory(os_path, model, path)
else:
raise web.HTTPError(
400, "Unhandled contents type: %s" % model['type'])
except web.HTTPError:
raise
except Exception as e:
self.log.error(u'Error while saving file: %s %s',
path,
e,
exc_info=True)
raise web.HTTPError(
500, u'Unexpected error while saving file: %s %s' % (path, e))
validation_message = None
if model['type'] == 'notebook':
self.validate_notebook_model(model)
validation_message = model.get('message', None)
model = self.get(path, content=False)
if validation_message:
model['message'] = validation_message
self.run_post_save_hook(model=model, os_path=os_path)
return model
def delete_file(self, path):
"""Delete file at path."""
path = path.strip('/')
os_path = self._get_os_path(path)
rm = os.unlink
if os.path.isdir(os_path):
listing = os.listdir(os_path)
# Don't delete non-empty directories.
# A directory containing only leftover checkpoints is
# considered empty.
cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None)
for entry in listing:
if entry != cp_dir:
raise web.HTTPError(400,
u'Directory %s not empty' % os_path)
elif not os.path.isfile(os_path):
raise web.HTTPError(404, u'File does not exist: %s' % os_path)
if os.path.isdir(os_path):
self.log.debug("Removing directory %s", os_path)
with self.perm_to_403():
shutil.rmtree(os_path)
else:
self.log.debug("Unlinking file %s", os_path)
with self.perm_to_403():
rm(os_path)
def rename_file(self, old_path, new_path):
"""Rename a file."""
old_path = old_path.strip('/')
new_path = new_path.strip('/')
if new_path == old_path:
return
new_os_path = self._get_os_path(new_path)
old_os_path = self._get_os_path(old_path)
# Should we proceed with the move?
if os.path.exists(new_os_path):
raise web.HTTPError(409, u'File already exists: %s' % new_path)
# Move the file
try:
with self.perm_to_403():
shutil.move(old_os_path, new_os_path)
except web.HTTPError:
raise
except Exception as e:
raise web.HTTPError(
500, u'Unknown error renaming file: %s %s' % (old_path, e))
def info_string(self):
return "Serving notebooks from local directory: %s" % self.root_dir
def get_kernel_path(self, path, model=None):
"""Return the initial API path of a kernel associated with a given notebook"""
if '/' in path:
parent_dir = path.rsplit('/', 1)[0]
else:
parent_dir = ''
return parent_dir
class IPClusterStart(IPClusterEngines):
name = u'ipcluster'
description = start_help
examples = _start_examples
default_log_level = logging.INFO
auto_create = Bool(
True,
config=True,
help="whether to create the profile_dir if it doesn't exist")
classes = List()
def _classes_default(self, ):
from IPython.parallel.apps import launcher
return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers
clean_logs = Bool(True,
config=True,
help="whether to cleanup old logs before starting")
delay = CFloat(
1.,
config=True,
help="delay (in s) between starting the controller and the engines")
controller_launcher_class = DottedObjectName(
'LocalControllerLauncher',
config=True,
helep=
"""The class for launching a Controller. Change this value if you want
your controller to also be launched by a batch system, such as PBS,SGE,MPIExec,etc.
Each launcher class has its own set of configuration options, for making sure
it will work in your environment.
Examples include:
LocalControllerLauncher : start engines locally as subprocesses
MPIExecControllerLauncher : use mpiexec to launch engines in an MPI universe
PBSControllerLauncher : use PBS (qsub) to submit engines to a batch queue
SGEControllerLauncher : use SGE (qsub) to submit engines to a batch queue
SSHControllerLauncher : use SSH to start the controller
WindowsHPCControllerLauncher : use Windows HPC
""")
reset = Bool(False,
config=True,
help="Whether to reset config files as part of '--create'.")
# flags = Dict(flags)
aliases = Dict(start_aliases)
def init_launchers(self):
self.controller_launcher = self.build_launcher(
self.controller_launcher_class)
self.engine_launcher = self.build_launcher(self.engine_launcher_class)
self.controller_launcher.on_stop(self.stop_launchers)
def start_controller(self):
self.controller_launcher.start(self.profile_dir.location)
def stop_controller(self):
# self.log.info("In stop_controller")
if self.controller_launcher and self.controller_launcher.running:
return self.controller_launcher.stop()
def stop_launchers(self, r=None):
if not self._stopping:
self.stop_controller()
super(IPClusterStart, self).stop_launchers()
def start(self):
"""Start the app for the start subcommand."""
# First see if the cluster is already running
try:
pid = self.get_pid_from_file()
except PIDFileError:
pass
else:
if self.check_pid(pid):
self.log.critical('Cluster is already running with [pid=%s]. '
'use "ipcluster stop" to stop the cluster.' %
pid)
# Here I exit with a unusual exit status that other processes
# can watch for to learn how I existed.
self.exit(ALREADY_STARTED)
else:
self.remove_pid_file()
# Now log and daemonize
self.log.info('Starting ipcluster with [daemon=%r]' % self.daemonize)
# TODO: Get daemonize working on Windows or as a Windows Server.
if self.daemonize:
if os.name == 'posix':
daemonize()
dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop)
dc.start()
dc = ioloop.DelayedCallback(self.start_engines, 1000 * self.delay,
self.loop)
dc.start()
# Now write the new pid file AFTER our new forked pid is active.
self.write_pid_file()
try:
self.loop.start()
except KeyboardInterrupt:
pass
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
pass
else:
raise
finally:
self.remove_pid_file()
class IPClusterEngines(BaseParallelApplication):
name = u'ipcluster'
description = engines_help
examples = _engines_examples
usage = None
config_file_name = Unicode(default_config_file_name)
default_log_level = logging.INFO
classes = List()
def _classes_default(self):
from IPython.parallel.apps import launcher
launchers = launcher.all_launchers
eslaunchers = [l for l in launchers if 'EngineSet' in l.__name__]
return [ProfileDir] + eslaunchers
n = Int(
num_cpus(),
config=True,
help=
"""The number of engines to start. The default is to use one for each
CPU on your machine""")
engine_launcher_class = DottedObjectName(
'LocalEngineSetLauncher',
config=True,
help="""The class for launching a set of Engines. Change this value
to use various batch systems to launch your engines, such as PBS,SGE,MPIExec,etc.
Each launcher class has its own set of configuration options, for making sure
it will work in your environment.
You can also write your own launcher, and specify it's absolute import path,
as in 'mymodule.launcher.FTLEnginesLauncher`.
Examples include:
LocalEngineSetLauncher : start engines locally as subprocesses [default]
MPIExecEngineSetLauncher : use mpiexec to launch in an MPI environment
PBSEngineSetLauncher : use PBS (qsub) to submit engines to a batch queue
SGEEngineSetLauncher : use SGE (qsub) to submit engines to a batch queue
SSHEngineSetLauncher : use SSH to start the controller
Note that SSH does *not* move the connection files
around, so you will likely have to do this manually
unless the machines are on a shared file system.
WindowsHPCEngineSetLauncher : use Windows HPC
""")
daemonize = Bool(
False,
config=True,
help="""Daemonize the ipcluster program. This implies --log-to-file.
Not available on Windows.
""")
def _daemonize_changed(self, name, old, new):
if new:
self.log_to_file = True
aliases = Dict(engine_aliases)
flags = Dict(engine_flags)
_stopping = False
def initialize(self, argv=None):
super(IPClusterEngines, self).initialize(argv)
self.init_signal()
self.init_launchers()
def init_launchers(self):
self.engine_launcher = self.build_launcher(self.engine_launcher_class)
self.engine_launcher.on_stop(lambda r: self.loop.stop())
def init_signal(self):
# Setup signals
signal.signal(signal.SIGINT, self.sigint_handler)
def build_launcher(self, clsname):
"""import and instantiate a Launcher based on importstring"""
if '.' not in clsname:
# not a module, presume it's the raw name in apps.launcher
clsname = 'IPython.parallel.apps.launcher.' + clsname
# print repr(clsname)
try:
klass = import_item(clsname)
except (ImportError, KeyError):
self.log.fatal("Could not import launcher class: %r" % clsname)
self.exit(1)
launcher = klass(work_dir=u'.', config=self.config, log=self.log)
return launcher
def start_engines(self):
self.log.info("Starting %i engines" % self.n)
self.engine_launcher.start(self.n, self.profile_dir.location)
def stop_engines(self):
self.log.info("Stopping Engines...")
if self.engine_launcher.running:
d = self.engine_launcher.stop()
return d
else:
return None
def stop_launchers(self, r=None):
if not self._stopping:
self._stopping = True
self.log.error("IPython cluster: stopping")
self.stop_engines()
# Wait a few seconds to let things shut down.
dc = ioloop.DelayedCallback(self.loop.stop, 4000, self.loop)
dc.start()
def sigint_handler(self, signum, frame):
self.log.debug("SIGINT received, stopping launchers...")
self.stop_launchers()
def start_logging(self):
# Remove old log files of the controller and engine
if self.clean_logs:
log_dir = self.profile_dir.log_dir
for f in os.listdir(log_dir):
if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)', f):
os.remove(os.path.join(log_dir, f))
# This will remove old log files for ipcluster itself
# super(IPBaseParallelApplication, self).start_logging()
def start(self):
"""Start the app for the engines subcommand."""
self.log.info("IPython cluster: started")
# First see if the cluster is already running
# Now log and daemonize
self.log.info('Starting engines with [daemon=%r]' % self.daemonize)
# TODO: Get daemonize working on Windows or as a Windows Server.
if self.daemonize:
if os.name == 'posix':
daemonize()
dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop)
dc.start()
# Now write the new pid file AFTER our new forked pid is active.
# self.write_pid_file()
try:
self.loop.start()
except KeyboardInterrupt:
pass
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
pass
else:
raise
class InteractiveShellApp(Configurable):
"""A Mixin for applications that start InteractiveShell instances.
Provides configurables for loading extensions and executing files
as part of configuring a Shell environment.
Provides init_extensions() and init_code() methods, to be called
after init_shell(), which must be implemented by subclasses.
"""
extensions = List(
Unicode,
config=True,
help="A list of dotted module names of IPython extensions to load.")
extra_extension = Unicode(
'',
config=True,
help="dotted module name of an IPython extension to load.")
def _extra_extension_changed(self, name, old, new):
if new:
# add to self.extensions
self.extensions.append(new)
exec_files = List(Unicode,
config=True,
help="""List of files to run at IPython startup.""")
file_to_run = Unicode('', config=True, help="""A file to be run""")
exec_lines = List(Unicode,
config=True,
help="""lines of code to run at IPython startup.""")
code_to_run = Unicode('',
config=True,
help="Execute the given command string.")
pylab_import_all = Bool(
True,
config=True,
help="""If true, an 'import *' is done from numpy and pylab,
when using pylab""")
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
def init_shell(self):
raise NotImplementedError("Override in subclasses")
def init_extensions(self):
"""Load all IPython extensions in IPythonApp.extensions.
This uses the :meth:`ExtensionManager.load_extensions` to load all
the extensions listed in ``self.extensions``.
"""
if not self.extensions:
return
try:
self.log.debug("Loading IPython extensions...")
extensions = self.extensions
for ext in extensions:
try:
self.log.info("Loading IPython extension: %s" % ext)
self.shell.extension_manager.load_extension(ext)
except:
self.log.warn("Error in loading extension: %s" % ext)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in loading extensions:")
self.shell.showtraceback()
def init_code(self):
"""run the pre-flight code, specified via exec_lines"""
self._run_startup_files()
self._run_exec_lines()
self._run_exec_files()
self._run_cmd_line_code()
def _run_exec_lines(self):
"""Run lines of code in IPythonApp.exec_lines in the user's namespace."""
if not self.exec_lines:
return
try:
self.log.debug("Running code from IPythonApp.exec_lines...")
for line in self.exec_lines:
try:
self.log.info("Running code in user namespace: %s" % line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user "
"namespace: %s" % line)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
self.shell.showtraceback()
def _exec_file(self, fname):
try:
full_filename = filefind(fname, [u'.', self.ipython_dir])
except IOError as e:
self.log.warn("File not found: %r" % fname)
return
# Make sure that the running script gets a proper sys.argv as if it
# were run from a system shell.
save_argv = sys.argv
sys.argv = [full_filename] + self.extra_args[1:]
# protect sys.argv from potential unicode strings on Python 2:
if not py3compat.PY3:
sys.argv = [py3compat.cast_bytes(a) for a in sys.argv]
try:
if os.path.isfile(full_filename):
if full_filename.endswith('.ipy'):
self.log.info("Running file in user namespace: %s" %
full_filename)
self.shell.safe_execfile_ipy(full_filename)
else:
# default to python, even without extension
self.log.info("Running file in user namespace: %s" %
full_filename)
# Ensure that __file__ is always defined to match Python behavior
self.shell.user_ns['__file__'] = fname
try:
self.shell.safe_execfile(full_filename,
self.shell.user_ns)
finally:
del self.shell.user_ns['__file__']
finally:
sys.argv = save_argv
def _run_startup_files(self):
"""Run files from profile startup directory"""
startup_dir = self.profile_dir.startup_dir
startup_files = glob.glob(os.path.join(startup_dir, '*.py'))
startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
if not startup_files:
return
self.log.debug("Running startup files from %s...", startup_dir)
try:
for fname in sorted(startup_files):
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling startup files:")
self.shell.showtraceback()
def _run_exec_files(self):
"""Run files from IPythonApp.exec_files"""
if not self.exec_files:
return
self.log.debug("Running files in IPythonApp.exec_files...")
try:
for fname in self.exec_files:
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling IPythonApp.exec_files:")
self.shell.showtraceback()
def _run_cmd_line_code(self):
"""Run code or file specified at the command-line"""
if self.code_to_run:
line = self.code_to_run
try:
self.log.info("Running code given at command line (c=): %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user namespace: %s" %
line)
self.shell.showtraceback()
# Like Python itself, ignore the second if the first of these is present
elif self.file_to_run:
fname = self.file_to_run
try:
self._exec_file(fname)
except:
self.log.warn("Error in executing file in user namespace: %s" %
fname)
self.shell.showtraceback()
class NotebookManager(LoggingConfigurable):
notebook_dir = Unicode(os.getcwdu(),
config=True,
help="""
The directory to use for notebooks.
""")
save_script = Bool(
False,
config=True,
help="""Automatically create a Python script when saving the notebook.
For easier use of import, %run and %load across notebooks, a
<notebook-name>.py script will be created next to any
<notebook-name>.ipynb on each save. This can also be set with the
short `--script` flag.
""")
filename_ext = Unicode(u'.ipynb')
allowed_formats = List([u'json', u'py'])
# Map notebook_ids to notebook names
mapping = Dict()
# Map notebook names to notebook_ids
rev_mapping = Dict()
def list_notebooks(self):
"""List all notebooks in the notebook dir.
This returns a list of dicts of the form::
dict(notebook_id=notebook,name=name)
"""
names = glob.glob(
os.path.join(self.notebook_dir, '*' + self.filename_ext))
names = [os.path.splitext(os.path.basename(name))[0] for name in names]
data = []
for name in names:
if name not in self.rev_mapping:
notebook_id = self.new_notebook_id(name)
else:
notebook_id = self.rev_mapping[name]
data.append(dict(notebook_id=notebook_id, name=name))
data = sorted(data, key=lambda item: item['name'])
return data
def new_notebook_id(self, name):
"""Generate a new notebook_id for a name and store its mappings."""
# TODO: the following will give stable urls for notebooks, but unless
# the notebooks are immediately redirected to their new urls when their
# filemname changes, nasty inconsistencies result. So for now it's
# disabled and instead we use a random uuid4() call. But we leave the
# logic here so that we can later reactivate it, whhen the necessary
# url redirection code is written.
#notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,
# 'file://'+self.get_path_by_name(name).encode('utf-8')))
notebook_id = unicode(uuid.uuid4())
self.mapping[notebook_id] = name
self.rev_mapping[name] = notebook_id
return notebook_id
def delete_notebook_id(self, notebook_id):
"""Delete a notebook's id only. This doesn't delete the actual notebook."""
name = self.mapping[notebook_id]
del self.mapping[notebook_id]
del self.rev_mapping[name]
def notebook_exists(self, notebook_id):
"""Does a notebook exist?"""
if notebook_id not in self.mapping:
return False
path = self.get_path_by_name(self.mapping[notebook_id])
return os.path.isfile(path)
def find_path(self, notebook_id):
"""Return a full path to a notebook given its notebook_id."""
try:
name = self.mapping[notebook_id]
except KeyError:
raise web.HTTPError(404,
u'Notebook does not exist: %s' % notebook_id)
return self.get_path_by_name(name)
def get_path_by_name(self, name):
"""Return a full path to a notebook given its name."""
filename = name + self.filename_ext
path = os.path.join(self.notebook_dir, filename)
return path
def get_notebook(self, notebook_id, format=u'json'):
"""Get the representation of a notebook in format by notebook_id."""
format = unicode(format)
if format not in self.allowed_formats:
raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
last_modified, nb = self.get_notebook_object(notebook_id)
kwargs = {}
if format == 'json':
# don't split lines for sending over the wire, because it
# should match the Python in-memory format.
kwargs['split_lines'] = False
data = current.writes(nb, format, **kwargs)
name = nb.get('name', 'notebook')
return last_modified, name, data
def get_notebook_object(self, notebook_id):
"""Get the NotebookNode representation of a notebook by notebook_id."""
path = self.find_path(notebook_id)
if not os.path.isfile(path):
raise web.HTTPError(404,
u'Notebook does not exist: %s' % notebook_id)
info = os.stat(path)
last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime)
with open(path, 'r') as f:
s = f.read()
try:
# v1 and v2 and json in the .ipynb files.
nb = current.reads(s, u'json')
except:
raise web.HTTPError(500, u'Unreadable JSON notebook.')
# Always use the filename as the notebook name.
nb.metadata.name = os.path.split(path)[-1].split(u'.')[0]
return last_modified, nb
def save_new_notebook(self, data, name=None, format=u'json'):
"""Save a new notebook and return its notebook_id.
If a name is passed in, it overrides any values in the notebook data
and the value in the data is updated to use that value.
"""
if format not in self.allowed_formats:
raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
try:
nb = current.reads(data.decode('utf-8'), format)
except:
raise web.HTTPError(400, u'Invalid JSON data')
if name is None:
try:
name = nb.metadata.name
except AttributeError:
raise web.HTTPError(400, u'Missing notebook name')
nb.metadata.name = name
notebook_id = self.new_notebook_id(name)
self.save_notebook_object(notebook_id, nb)
return notebook_id
def save_notebook(self, notebook_id, data, name=None, format=u'json'):
"""Save an existing notebook by notebook_id."""
if format not in self.allowed_formats:
raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
try:
nb = current.reads(data.decode('utf-8'), format)
except:
raise web.HTTPError(400, u'Invalid JSON data')
if name is not None:
nb.metadata.name = name
self.save_notebook_object(notebook_id, nb)
def save_notebook_object(self, notebook_id, nb):
"""Save an existing notebook object by notebook_id."""
if notebook_id not in self.mapping:
raise web.HTTPError(404,
u'Notebook does not exist: %s' % notebook_id)
old_name = self.mapping[notebook_id]
try:
new_name = nb.metadata.name
except AttributeError:
raise web.HTTPError(400, u'Missing notebook name')
path = self.get_path_by_name(new_name)
try:
with open(path, 'w') as f:
current.write(nb, f, u'json')
except Exception as e:
raise web.HTTPError(
400, u'Unexpected error while saving notebook: %s' % e)
# save .py script as well
if self.save_script:
pypath = os.path.splitext(path)[0] + '.py'
try:
with io.open(pypath, 'w', encoding='utf-8') as f:
current.write(nb, f, u'py')
except Exception as e:
raise web.HTTPError(
400,
u'Unexpected error while saving notebook as script: %s' %
e)
if old_name != new_name:
old_path = self.get_path_by_name(old_name)
if os.path.isfile(old_path):
os.unlink(old_path)
if self.save_script:
old_pypath = os.path.splitext(old_path)[0] + '.py'
if os.path.isfile(old_pypath):
os.unlink(old_pypath)
self.mapping[notebook_id] = new_name
self.rev_mapping[new_name] = notebook_id
def delete_notebook(self, notebook_id):
"""Delete notebook by notebook_id."""
path = self.find_path(notebook_id)
if not os.path.isfile(path):
raise web.HTTPError(404,
u'Notebook does not exist: %s' % notebook_id)
os.unlink(path)
self.delete_notebook_id(notebook_id)
def increment_filename(self, basename):
"""Return a non-used filename of the form basename<int>.
This searches through the filenames (basename0, basename1, ...)
until is find one that is not already being used. It is used to
create Untitled and Copy names that are unique.
"""
i = 0
while True:
name = u'%s%i' % (basename, i)
path = self.get_path_by_name(name)
if not os.path.isfile(path):
break
else:
i = i + 1
return path, name
def new_notebook(self):
"""Create a new notebook and return its notebook_id."""
path, name = self.increment_filename('Untitled')
notebook_id = self.new_notebook_id(name)
metadata = current.new_metadata(name=name)
nb = current.new_notebook(metadata=metadata)
with open(path, 'w') as f:
current.write(nb, f, u'json')
return notebook_id
def copy_notebook(self, notebook_id):
"""Copy an existing notebook and return its notebook_id."""
last_mod, nb = self.get_notebook_object(notebook_id)
name = nb.metadata.name + '-Copy'
path, name = self.increment_filename(name)
nb.metadata.name = name
notebook_id = self.new_notebook_id(name)
self.save_notebook_object(notebook_id, nb)
return notebook_id
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
default_config_file_name = 'ipython_notebook_config.py'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = IPythonConsoleApp.classes + [
MappingKernelManager, NotebookManager, FileNotebookManager
]
flags = Dict(flags)
aliases = Dict(aliases)
kernel_argv = List(Unicode)
max_msg_size = Integer(65536,
config=True,
help="""
The max raw message size accepted from the browser
over a WebSocket connection.
""")
def _log_level_default(self):
return logging.INFO
def _log_format_default(self):
"""override default log format to include time"""
return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s"
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('')
# Network related information.
ip = Unicode(LOCALHOST,
config=True,
help="The IP address the notebook server will listen on.")
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888,
config=True,
help="The port the notebook server will listen on.")
port_retries = Integer(
50,
config=True,
help=
"The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(
u'',
config=True,
help="""The full path to an SSL/TLS certificate file.""")
keyfile = Unicode(
u'',
config=True,
help="""The full path to a private key file for usage with SSL/TLS.""")
password = Unicode(u'',
config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
""")
open_browser = Bool(True,
config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'',
config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
read_only = Bool(False,
config=True,
help="Whether to prevent editing/execution of notebooks.")
use_less = Bool(False,
config=True,
help="""Wether to use Browser Side less-css parsing
instead of compiled css version in templates that allows
it. This is mainly convenient when working on the less
file to avoid a build step, or if user want to overwrite
some of the less variables without having to recompile
everything.
You will need to install the less.js component in the static directory
either in the source tree or in your profile folder.
""")
webapp_settings = Dict(
config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
enable_mathjax = Bool(
True,
config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
""")
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_project_url = Unicode('/',
config=True,
help='''The base URL for the notebook server.
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_project_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_project_url = '/' + new
elif not new.endswith('/'):
self.base_project_url = new + '/'
base_kernel_url = Unicode('/',
config=True,
help='''The base URL for the kernel server
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_kernel_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_kernel_url = '/' + new
elif not new.endswith('/'):
self.base_kernel_url = new + '/'
websocket_url = Unicode("",
config=True,
help="""The base URL for the websocket server,
if it differs from the HTTP server (hint: it almost certainly doesn't).
Should be in the form of an HTTP origin: ws[s]://hostname[:port]
""")
extra_static_paths = List(
Unicode,
config=True,
help="""Extra paths to search for serving static files.
This allows adding javascript/css to be available from the notebook server machine,
or overriding individual files in the IPython""")
def _extra_static_paths_default(self):
return [os.path.join(self.profile_dir.location, 'static')]
@property
def static_file_path(self):
"""return extra paths + the default location"""
return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""")
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_url_prefix = self.webapp_settings.get("static_url_prefix",
"/static/")
try:
mathjax = filefind(os.path.join('mathjax', 'MathJax.js'),
self.static_file_path)
except IOError:
if self.certfile:
# HTTPS: load from Rackspace CDN, because SSL certificate requires it
base = u"https://c328740.ssl.cf1.rackcdn.com"
else:
base = u"http://cdn.mathjax.org"
url = base + u"/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
else:
self.log.info("Using local MathJax from %s" % mathjax)
return static_url_prefix + u"mathjax/MathJax.js"
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
notebook_manager_class = DottedObjectName(
'IPython.frontend.html.notebook.services.notebooks.filenbmanager.FileNotebookManager',
config=True,
help='The notebook manager class to use.')
trust_xheaders = Bool(
False,
config=True,
help=
("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
"sent by the upstream reverse proxy. Neccesary if the proxy handles SSL"
))
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if argv is None:
argv = sys.argv[1:]
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" %
self.name)
if self.extra_args:
f = os.path.abspath(self.extra_args[0])
if os.path.isdir(f):
nbdir = f
else:
self.file_to_run = f
nbdir = os.path.dirname(f)
self.config.NotebookManager.notebook_dir = nbdir
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_manager = MappingKernelManager(
config=self.config,
log=self.log,
kernel_argv=self.kernel_argv,
connection_dir=self.profile_dir.security_dir,
)
kls = import_item(self.notebook_manager_class)
self.notebook_manager = kls(config=self.config, log=self.log)
self.notebook_manager.load_notebook_names()
self.cluster_manager = ClusterManager(config=self.config, log=self.log)
self.cluster_manager.update_profiles()
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
# hook up tornado 3's loggers to our app handlers
for name in ('access', 'application', 'general'):
logging.getLogger('tornado.%s' % name).handlers = self.log.handlers
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(self, self.kernel_manager,
self.notebook_manager,
self.cluster_manager, self.log,
self.base_project_url,
self.webapp_settings)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app,
ssl_options=ssl_options,
xheaders=self.trust_xheaders)
if not self.ip:
warning = "WARNING: The notebook server is listening on all IP addresses"
if ssl_options is None:
self.log.critical(warning + " and not using encryption. This "
"is not recommended.")
if not self.password and not self.read_only:
self.log.critical(
warning + " and not using authentication. "
"This is highly insecure and not recommended.")
success = None
for port in random_ports(self.port, self.port_retries + 1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
# XXX: remove the e.errno == -9 block when we require
# tornado >= 3.0
if e.errno == -9 and tornado.version_info[0] < 3:
# The flags passed to socket.getaddrinfo from
# tornado.netutils.bind_sockets can cause "gaierror:
# [Errno -9] Address family for hostname not supported"
# when the interface is not associated, for example.
# Changing the flags to exclude socket.AI_ADDRCONFIG does
# not cause this error, but the only way to do this is to
# monkeypatch socket to remove the AI_ADDRCONFIG attribute
saved_AI_ADDRCONFIG = socket.AI_ADDRCONFIG
self.log.warn('Monkeypatching socket to fix tornado bug')
del (socket.AI_ADDRCONFIG)
try:
# retry the tornado call without AI_ADDRCONFIG flags
self.http_server.listen(port, self.ip)
except socket.error as e2:
e = e2
else:
self.port = port
success = True
break
# restore the monekypatch
socket.AI_ADDRCONFIG = saved_AI_ADDRCONFIG
if e.errno != errno.EADDRINUSE:
raise
self.log.info(
'The port %i is already in use, trying another random port.'
% port)
else:
self.port = port
success = True
break
if not success:
self.log.critical(
'ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
def init_signal(self):
if not sys.platform.startswith('win'):
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
if hasattr(signal, 'SIGUSR1'):
# Windows doesn't support SIGUSR1
signal.signal(signal.SIGUSR1, self._signal_info)
if hasattr(signal, 'SIGINFO'):
# only on BSD-based systems
signal.signal(signal.SIGINFO, self._signal_info)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
# FIXME: remove this delay when pyzmq dependency is >= 2.1.11
time.sleep(0.1)
info = self.log.info
info('interrupted')
print self.notebook_info()
sys.stdout.write("Shutdown this notebook server (y/[n])? ")
sys.stdout.flush()
r, w, x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y'):
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.instance().stop()
return
else:
print "No answer for 5s:",
print "resuming operation..."
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.instance().stop()
def _signal_info(self, sig, frame):
print self.notebook_info()
def init_components(self):
"""Check the components submodule, and warn if it's unclean"""
status = submodule.check_submodule_status()
if status == 'missing':
self.log.warn(
"components submodule missing, running `git submodule update`")
submodule.update_submodules(submodule.ipython_parent())
elif status == 'unclean':
self.log.warn(
"components submodule unclean, you may see 404s on static/components"
)
self.log.warn(
"run `setup.py submodule` or `git submodule update` to update")
@catch_config_error
def initialize(self, argv=None):
self.init_logging()
super(NotebookApp, self).initialize(argv)
self.init_configurables()
self.init_components()
self.init_webapp()
self.init_signal()
def cleanup_kernels(self):
"""Shutdown all kernels.
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
self.kernel_manager.shutdown_all()
def notebook_info(self):
"Return the current working directory and the server url information"
mgr_info = self.notebook_manager.info_string() + "\n"
return mgr_info + "The IPython Notebook is running at: %s" % self._url
def start(self):
""" Start the IPython Notebook server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
ip = self.ip if self.ip else '[all ip addresses on your system]'
proto = 'https' if self.certfile else 'http'
info = self.log.info
self._url = "%s://%s:%i%s" % (proto, ip, self.port,
self.base_project_url)
for line in self.notebook_info().split("\n"):
info(line)
info("Use Control-C to stop this server and shut down all kernels.")
if self.open_browser or self.file_to_run:
ip = self.ip or LOCALHOST
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
name, _ = os.path.splitext(os.path.basename(self.file_to_run))
url = self.notebook_manager.rev_mapping.get(name, '')
else:
url = ''
if browser:
b = lambda: browser.open("%s://%s:%i%s%s" % (
proto, ip, self.port, self.base_project_url, url),
new=2)
threading.Thread(target=b).start()
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
aliases = aliases
flags = flags
classes = [
KernelManager, ProfileDir, Session, MappingKernelManager,
ContentsManager, FileContentsManager, NotebookNotary,
KernelSpecManager,
]
flags = Dict(flags)
aliases = Dict(aliases)
subcommands = dict(
list=(NbserverListApp, NbserverListApp.description.splitlines()[0]),
)
ipython_kernel_argv = List(Unicode)
_log_formatter_cls = LogFormatter
def _log_level_default(self):
return logging.INFO
def _log_datefmt_default(self):
"""Exclude date from default date format"""
return "%H:%M:%S"
def _log_format_default(self):
"""override default log format to include time"""
return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s"
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('', config=True)
# Network related information
allow_origin = Unicode('', config=True,
help="""Set the Access-Control-Allow-Origin header
Use '*' to allow any origin to access your server.
Takes precedence over allow_origin_pat.
"""
)
allow_origin_pat = Unicode('', config=True,
help="""Use a regular expression for the Access-Control-Allow-Origin header
Requests from an origin matching the expression will get replies with:
Access-Control-Allow-Origin: origin
where `origin` is the origin of the request.
Ignored if allow_origin is set.
"""
)
allow_credentials = Bool(False, config=True,
help="Set the Access-Control-Allow-Credentials: true header"
)
default_url = Unicode('/tree', config=True,
help="The default URL to redirect to from `/`"
)
ip = Unicode('localhost', config=True,
help="The IP address the notebook server will listen on."
)
def _ip_default(self):
"""Return localhost if available, 127.0.0.1 otherwise.
On some (horribly broken) systems, localhost cannot be bound.
"""
s = socket.socket()
try:
s.bind(('localhost', 0))
except socket.error as e:
self.log.warn("Cannot bind to localhost, using 127.0.0.1 as default ip\n%s", e)
return '127.0.0.1'
else:
s.close()
return 'localhost'
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888, config=True,
help="The port the notebook server will listen on."
)
port_retries = Integer(50, config=True,
help="The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(u'', config=True,
help="""The full path to an SSL/TLS certificate file."""
)
keyfile = Unicode(u'', config=True,
help="""The full path to a private key file for usage with SSL/TLS."""
)
cookie_secret_file = Unicode(config=True,
help="""The file where the cookie secret is stored."""
)
def _cookie_secret_file_default(self):
if self.profile_dir is None:
return ''
return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret')
cookie_secret = Bytes(b'', config=True,
help="""The random bytes used to secure cookies.
By default this is a new random number every time you start the Notebook.
Set it to a value in a config file to enable logins to persist across server sessions.
Note: Cookie secrets should be kept private, do not share config files with
cookie_secret stored in plaintext (you can read the value from a file).
"""
)
def _cookie_secret_default(self):
if os.path.exists(self.cookie_secret_file):
with io.open(self.cookie_secret_file, 'rb') as f:
return f.read()
else:
secret = base64.encodestring(os.urandom(1024))
self._write_cookie_secret_file(secret)
return secret
def _write_cookie_secret_file(self, secret):
"""write my secret to my secret_file"""
self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file)
with io.open(self.cookie_secret_file, 'wb') as f:
f.write(secret)
try:
os.chmod(self.cookie_secret_file, 0o600)
except OSError:
self.log.warn(
"Could not set permissions on %s",
self.cookie_secret_file
)
password = Unicode(u'', config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
"""
)
open_browser = Bool(True, config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'', config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
webapp_settings = Dict(config=True,
help="DEPRECATED, use tornado_settings"
)
def _webapp_settings_changed(self, name, old, new):
self.log.warn("\n webapp_settings is deprecated, use tornado_settings.\n")
self.tornado_settings = new
tornado_settings = Dict(config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
ssl_options = Dict(config=True,
help="""Supply SSL options for the tornado HTTPServer.
See the tornado docs for details.""")
jinja_environment_options = Dict(config=True,
help="Supply extra arguments that will be passed to Jinja environment.")
jinja_template_vars = Dict(
config=True,
help="Extra variables to supply to jinja templates when rendering.",
)
enable_mathjax = Bool(True, config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
"""
)
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_url = Unicode('/', config=True,
help='''The base URL for the notebook server.
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_url = '/'+new
elif not new.endswith('/'):
self.base_url = new+'/'
base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""")
def _base_project_url_changed(self, name, old, new):
self.log.warn("base_project_url is deprecated, use base_url")
self.base_url = new
extra_static_paths = List(Unicode, config=True,
help="""Extra paths to search for serving static files.
This allows adding javascript/css to be available from the notebook server machine,
or overriding individual files in the IPython"""
)
def _extra_static_paths_default(self):
return [os.path.join(self.profile_dir.location, 'static')]
@property
def static_file_path(self):
"""return extra paths + the default location"""
return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
extra_template_paths = List(Unicode, config=True,
help="""Extra paths to search for serving jinja templates.
Can be used to override templates from IPython.html.templates."""
)
def _extra_template_paths_default(self):
return []
@property
def template_file_path(self):
"""return extra paths + the default locations"""
return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST
extra_nbextensions_path = List(Unicode, config=True,
help="""extra paths to look for Javascript notebook extensions"""
)
@property
def nbextensions_path(self):
"""The path to look for Javascript notebook extensions"""
return self.extra_nbextensions_path + [os.path.join(get_ipython_dir(), 'nbextensions')] + SYSTEM_NBEXTENSIONS_DIRS
websocket_url = Unicode("", config=True,
help="""The base URL for websockets,
if it differs from the HTTP server (hint: it almost certainly doesn't).
Should be in the form of an HTTP origin: ws[s]://hostname[:port]
"""
)
mathjax_url = Unicode("", config=True,
help="""The url for MathJax.js."""
)
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_url_prefix = self.tornado_settings.get("static_url_prefix",
url_path_join(self.base_url, "static")
)
# try local mathjax, either in nbextensions/mathjax or static/mathjax
for (url_prefix, search_path) in [
(url_path_join(self.base_url, "nbextensions"), self.nbextensions_path),
(static_url_prefix, self.static_file_path),
]:
self.log.debug("searching for local mathjax in %s", search_path)
try:
mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path)
except IOError:
continue
else:
url = url_path_join(url_prefix, u"mathjax/MathJax.js")
self.log.info("Serving local MathJax from %s at %s", mathjax, url)
return url
# no local mathjax, serve from CDN
url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
contents_manager_class = Type(
default_value=FileContentsManager,
klass=ContentsManager,
config=True,
help='The notebook manager class to use.'
)
kernel_manager_class = Type(
default_value=MappingKernelManager,
config=True,
help='The kernel manager class to use.'
)
session_manager_class = Type(
default_value=SessionManager,
config=True,
help='The session manager class to use.'
)
cluster_manager_class = Type(
default_value=ClusterManager,
config=True,
help='The cluster manager class to use.'
)
config_manager_class = Type(
default_value=ConfigManager,
config = True,
help='The config manager class to use'
)
kernel_spec_manager = Instance(KernelSpecManager)
kernel_spec_manager_class = Type(
default_value=KernelSpecManager,
config=True,
help="""
The kernel spec manager class to use. Should be a subclass
of `IPython.kernel.kernelspec.KernelSpecManager`.
The Api of KernelSpecManager is provisional and might change
without warning between this version of IPython and the next stable one.
"""
)
login_handler_class = Type(
default_value=LoginHandler,
klass=web.RequestHandler,
config=True,
help='The login handler class to use.',
)
logout_handler_class = Type(
default_value=LogoutHandler,
klass=web.RequestHandler,
config=True,
help='The logout handler class to use.',
)
trust_xheaders = Bool(False, config=True,
help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
"sent by the upstream reverse proxy. Necessary if the proxy handles SSL")
)
info_file = Unicode()
def _info_file_default(self):
info_file = "nbserver-%s.json"%os.getpid()
return os.path.join(self.profile_dir.security_dir, info_file)
pylab = Unicode('disabled', config=True,
help="""
DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.
"""
)
def _pylab_changed(self, name, old, new):
"""when --pylab is specified, display a warning and exit"""
if new != 'warn':
backend = ' %s' % new
else:
backend = ''
self.log.error("Support for specifying --pylab on the command line has been removed.")
self.log.error(
"Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself.".format(backend)
)
self.exit(1)
notebook_dir = Unicode(config=True,
help="The directory to use for notebooks and kernels."
)
def _notebook_dir_default(self):
if self.file_to_run:
return os.path.dirname(os.path.abspath(self.file_to_run))
else:
return py3compat.getcwd()
def _notebook_dir_changed(self, name, old, new):
"""Do a bit of validation of the notebook dir."""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
self.notebook_dir = os.path.abspath(new)
return
if not os.path.isdir(new):
raise TraitError("No such notebook dir: %r" % new)
# setting App.notebook_dir implies setting notebook and kernel dirs as well
self.config.FileContentsManager.root_dir = new
self.config.MappingKernelManager.root_dir = new
server_extensions = List(Unicode(), config=True,
help=("Python modules to load as notebook server extensions. "
"This is an experimental API, and may change in future releases.")
)
reraise_server_extension_failures = Bool(
False,
config=True,
help="Reraise exceptions encountered loading server extensions?",
)
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if self.extra_args:
arg0 = self.extra_args[0]
f = os.path.abspath(arg0)
self.argv.remove(arg0)
if not os.path.exists(f):
self.log.critical("No such file or directory: %s", f)
self.exit(1)
# Use config here, to ensure that it takes higher priority than
# anything that comes from the profile.
c = Config()
if os.path.isdir(f):
c.NotebookApp.notebook_dir = f
elif os.path.isfile(f):
c.NotebookApp.file_to_run = f
self.update_config(c)
def init_kernel_argv(self):
"""add the profile-dir to arguments to be passed to IPython kernels"""
# FIXME: remove special treatment of IPython kernels
# Kernel should get *absolute* path to profile directory
self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location]
def init_configurables(self):
self.kernel_spec_manager = self.kernel_spec_manager_class(
parent=self,
ipython_dir=self.ipython_dir,
)
self.kernel_manager = self.kernel_manager_class(
parent=self,
log=self.log,
ipython_kernel_argv=self.ipython_kernel_argv,
connection_dir=self.profile_dir.security_dir,
)
self.contents_manager = self.contents_manager_class(
parent=self,
log=self.log,
)
self.session_manager = self.session_manager_class(
parent=self,
log=self.log,
kernel_manager=self.kernel_manager,
contents_manager=self.contents_manager,
)
self.cluster_manager = self.cluster_manager_class(
parent=self,
log=self.log,
)
self.config_manager = self.config_manager_class(
parent=self,
log=self.log,
profile_dir=self.profile_dir.location,
)
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
for log in app_log, access_log, gen_log:
# consistent log output name (NotebookApp instead of tornado.access, etc.)
log.name = self.log.name
# hook up tornado 3's loggers to our app handlers
logger = logging.getLogger('tornado')
logger.propagate = True
logger.parent = self.log
logger.setLevel(self.log.level)
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.tornado_settings['allow_origin'] = self.allow_origin
if self.allow_origin_pat:
self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat)
self.tornado_settings['allow_credentials'] = self.allow_credentials
# ensure default_url starts with base_url
if not self.default_url.startswith(self.base_url):
self.default_url = url_path_join(self.base_url, self.default_url)
self.web_app = NotebookWebApplication(
self, self.kernel_manager, self.contents_manager,
self.cluster_manager, self.session_manager, self.kernel_spec_manager,
self.config_manager,
self.log, self.base_url, self.default_url, self.tornado_settings,
self.jinja_environment_options
)
ssl_options = self.ssl_options
if self.certfile:
ssl_options['certfile'] = self.certfile
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
if not ssl_options:
# None indicates no SSL config
ssl_options = None
else:
# Disable SSLv3, since its use is discouraged.
ssl_options['ssl_version']=ssl.PROTOCOL_TLSv1
self.login_handler_class.validate_security(self, ssl_options=ssl_options)
self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options,
xheaders=self.trust_xheaders)
success = None
for port in random_ports(self.port, self.port_retries+1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
if e.errno == errno.EADDRINUSE:
self.log.info('The port %i is already in use, trying another random port.' % port)
continue
elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)):
self.log.warn("Permission to listen on port %i denied" % port)
continue
else:
raise
else:
self.port = port
success = True
break
if not success:
self.log.critical('ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
@property
def display_url(self):
ip = self.ip if self.ip else '[all ip addresses on your system]'
return self._url(ip)
@property
def connection_url(self):
ip = self.ip if self.ip else 'localhost'
return self._url(ip)
def _url(self, ip):
proto = 'https' if self.certfile else 'http'
return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url)
def init_terminals(self):
try:
from .terminal import initialize
initialize(self.web_app)
self.web_app.settings['terminals_available'] = True
except ImportError as e:
log = self.log.debug if sys.platform == 'win32' else self.log.warn
log("Terminals not available (error was %s)", e)
def init_signal(self):
if not sys.platform.startswith('win'):
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
if hasattr(signal, 'SIGUSR1'):
# Windows doesn't support SIGUSR1
signal.signal(signal.SIGUSR1, self._signal_info)
if hasattr(signal, 'SIGINFO'):
# only on BSD-based systems
signal.signal(signal.SIGINFO, self._signal_info)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
info = self.log.info
info('interrupted')
print(self.notebook_info())
sys.stdout.write("Shutdown this notebook server (y/[n])? ")
sys.stdout.flush()
r,w,x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y') and 'n' not in line.lower():
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.current().stop()
return
else:
print("No answer for 5s:", end=' ')
print("resuming operation...")
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.current().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.current().stop()
def _signal_info(self, sig, frame):
print(self.notebook_info())
def init_components(self):
"""Check the components submodule, and warn if it's unclean"""
status = submodule.check_submodule_status()
if status == 'missing':
self.log.warn("components submodule missing, running `git submodule update`")
submodule.update_submodules(submodule.ipython_parent())
elif status == 'unclean':
self.log.warn("components submodule unclean, you may see 404s on static/components")
self.log.warn("run `setup.py submodule` or `git submodule update` to update")
def init_server_extensions(self):
"""Load any extensions specified by config.
Import the module, then call the load_jupyter_server_extension function,
if one exists.
The extension API is experimental, and may change in future releases.
"""
for modulename in self.server_extensions:
try:
mod = importlib.import_module(modulename)
func = getattr(mod, 'load_jupyter_server_extension', None)
if func is not None:
func(self)
except Exception:
if self.reraise_server_extension_failures:
raise
self.log.warn("Error loading server extension %s", modulename,
exc_info=True)
@catch_config_error
def initialize(self, argv=None):
super(NotebookApp, self).initialize(argv)
self.init_logging()
self.init_kernel_argv()
self.init_configurables()
self.init_components()
self.init_webapp()
self.init_terminals()
self.init_signal()
self.init_server_extensions()
def cleanup_kernels(self):
"""Shutdown all kernels.
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
self.kernel_manager.shutdown_all()
def notebook_info(self):
"Return the current working directory and the server url information"
info = self.contents_manager.info_string() + "\n"
info += "%d active kernels \n" % len(self.kernel_manager._kernels)
return info + "The IPython Notebook is running at: %s" % self.display_url
def server_info(self):
"""Return a JSONable dict of information about this server."""
return {'url': self.connection_url,
'hostname': self.ip if self.ip else 'localhost',
'port': self.port,
'secure': bool(self.certfile),
'base_url': self.base_url,
'notebook_dir': os.path.abspath(self.notebook_dir),
'pid': os.getpid()
}
def write_server_info_file(self):
"""Write the result of server_info() to the JSON file info_file."""
with open(self.info_file, 'w') as f:
json.dump(self.server_info(), f, indent=2)
def remove_server_info_file(self):
"""Remove the nbserver-<pid>.json file created for this server.
Ignores the error raised when the file has already been removed.
"""
try:
os.unlink(self.info_file)
except OSError as e:
if e.errno != errno.ENOENT:
raise
def start(self):
""" Start the IPython Notebook server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
if self.subapp is not None:
return self.subapp.start()
info = self.log.info
for line in self.notebook_info().split("\n"):
info(line)
info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")
self.write_server_info_file()
if self.open_browser or self.file_to_run:
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
if not os.path.exists(self.file_to_run):
self.log.critical("%s does not exist" % self.file_to_run)
self.exit(1)
relpath = os.path.relpath(self.file_to_run, self.notebook_dir)
uri = url_path_join('notebooks', *relpath.split(os.sep))
else:
uri = 'tree'
if browser:
b = lambda : browser.open(url_path_join(self.connection_url, uri),
new=2)
threading.Thread(target=b).start()
self.io_loop = ioloop.IOLoop.current()
if sys.platform.startswith('win'):
# add no-op to wake every 5s
# to handle signals that may be ignored by the inner loop
pc = ioloop.PeriodicCallback(lambda : None, 5000)
pc.start()
try:
self.io_loop.start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
self.remove_server_info_file()
def stop(self):
def _stop():
self.http_server.stop()
self.io_loop.stop()
self.io_loop.add_callback(_stop)
class DisplayFormatter(Configurable):
# When set to true only the default plain text formatter will be used.
plain_text_only = Bool(False, config=True)
def _plain_text_only_changed(self, name, old, new):
warnings.warn(
"""DisplayFormatter.plain_text_only is deprecated.
Use DisplayFormatter.active_types = ['text/plain']
for the same effect.
""", DeprecationWarning)
if new:
self.active_types = ['text/plain']
else:
self.active_types = self.format_types
active_types = List(Unicode,
config=True,
help="""List of currently active mime-types to display.
You can use this to set a white-list for formats to display.
Most users will not need to change this value.
""")
def _active_types_default(self):
return self.format_types
def _active_types_changed(self, name, old, new):
for key, formatter in self.formatters.items():
if key in new:
formatter.enabled = True
else:
formatter.enabled = False
# A dict of formatter whose keys are format types (MIME types) and whose
# values are subclasses of BaseFormatter.
formatters = Dict()
def _formatters_default(self):
"""Activate the default formatters."""
formatter_classes = [
PlainTextFormatter, HTMLFormatter, MarkdownFormatter, SVGFormatter,
PNGFormatter, PDFFormatter, JPEGFormatter, LatexFormatter,
JSONFormatter, JavascriptFormatter
]
d = {}
for cls in formatter_classes:
f = cls(parent=self)
d[f.format_type] = f
return d
def format(self, obj, include=None, exclude=None):
"""Return a format data dict for an object.
By default all format types will be computed.
The following MIME types are currently implemented:
* text/plain
* text/html
* text/markdown
* text/latex
* application/json
* application/javascript
* application/pdf
* image/png
* image/jpeg
* image/svg+xml
Parameters
----------
obj : object
The Python object whose format data will be computed.
include : list or tuple, optional
A list of format type strings (MIME types) to include in the
format data dict. If this is set *only* the format types included
in this list will be computed.
exclude : list or tuple, optional
A list of format type string (MIME types) to exclude in the format
data dict. If this is set all format types will be computed,
except for those included in this argument.
Returns
-------
(format_dict, metadata_dict) : tuple of two dicts
format_dict is a dictionary of key/value pairs, one of each format that was
generated for the object. The keys are the format types, which
will usually be MIME type strings and the values and JSON'able
data structure containing the raw data for the representation in
that format.
metadata_dict is a dictionary of metadata about each mime-type output.
Its keys will be a strict subset of the keys in format_dict.
"""
format_dict = {}
md_dict = {}
for format_type, formatter in self.formatters.items():
if include and format_type not in include:
continue
if exclude and format_type in exclude:
continue
md = None
try:
data = formatter(obj)
except:
# FIXME: log the exception
raise
# formatters can return raw data or (data, metadata)
if isinstance(data, tuple) and len(data) == 2:
data, md = data
if data is not None:
format_dict[format_type] = data
if md is not None:
md_dict[format_type] = md
return format_dict, md_dict
@property
def format_types(self):
"""Return the format types (MIME types) of the active formatters."""
return list(self.formatters.keys())
class _Selection(DOMWidget):
"""Base class for Selection widgets
``options`` can be specified as a list or dict. If given as a list,
it will be transformed to a dict of the form ``{str(value):value}``.
"""
value = Any(help="Selected value")
selected_label = Unicode(help="The label of the selected value", sync=True)
options = Any(
help="""List of (key, value) tuples or dict of values that the
user can select.
The keys of this list are the strings that will be displayed in the UI,
representing the actual Python choices.
The keys of this list are also available as _options_labels.
""")
_options_dict = Dict()
_options_labels = Tuple(sync=True)
_options_values = Tuple()
disabled = Bool(False, help="Enable or disable user changes", sync=True)
description = Unicode(
help="Description of the value this widget represents", sync=True)
def __init__(self, *args, **kwargs):
self.value_lock = Lock()
self.options_lock = Lock()
self.on_trait_change(self._options_readonly_changed, [
'_options_dict', '_options_labels', '_options_values', '_options'
])
if 'options' in kwargs:
self.options = kwargs.pop('options')
DOMWidget.__init__(self, *args, **kwargs)
self._value_in_options()
def _make_options(self, x):
# If x is a dict, convert it to list format.
if isinstance(x, (OrderedDict, dict)):
return [(k, v) for k, v in x.items()]
# Make sure x is a list or tuple.
if not isinstance(x, (list, tuple)):
raise ValueError('x')
# If x is an ordinary list, use the option values as names.
for y in x:
if not isinstance(y, (list, tuple)) or len(y) < 2:
return [(i, i) for i in x]
# Value is already in the correct format.
return x
def _options_changed(self, name, old, new):
"""Handles when the options tuple has been changed.
Setting options implies setting option labels from the keys of the dict.
"""
if self.options_lock.acquire(False):
try:
self.options = new
options = self._make_options(new)
self._options_dict = {i[0]: i[1] for i in options}
self._options_labels = [i[0] for i in options]
self._options_values = [i[1] for i in options]
self._value_in_options()
finally:
self.options_lock.release()
def _value_in_options(self):
# ensure that the chosen value is one of the choices
if self._options_values:
if self.value not in self._options_values:
self.value = next(iter(self._options_values))
def _options_readonly_changed(self, name, old, new):
if not self.options_lock.locked():
raise TraitError(
"`.%s` is a read-only trait. Use the `.options` tuple instead."
% name)
def _value_changed(self, name, old, new):
"""Called when value has been changed"""
if self.value_lock.acquire(False):
try:
# Reverse dictionary lookup for the value name
for k, v in self._options_dict.items():
if new == v:
# set the selected value name
self.selected_label = k
return
# undo the change, and raise KeyError
self.value = old
raise KeyError(new)
finally:
self.value_lock.release()
def _selected_label_changed(self, name, old, new):
"""Called when the value name has been changed (typically by the frontend)."""
if self.value_lock.acquire(False):
try:
self.value = self._options_dict[new]
finally:
self.value_lock.release()
class BaseFormatter(Configurable):
"""A base formatter class that is configurable.
This formatter should usually be used as the base class of all formatters.
It is a traited :class:`Configurable` class and includes an extensible
API for users to determine how their objects are formatted. The following
logic is used to find a function to format an given object.
1. The object is introspected to see if it has a method with the name
:attr:`print_method`. If is does, that object is passed to that method
for formatting.
2. If no print method is found, three internal dictionaries are consulted
to find print method: :attr:`singleton_printers`, :attr:`type_printers`
and :attr:`deferred_printers`.
Users should use these dictionaries to register functions that will be
used to compute the format data for their objects (if those objects don't
have the special print methods). The easiest way of using these
dictionaries is through the :meth:`for_type` and :meth:`for_type_by_name`
methods.
If no function/callable is found to compute the format data, ``None`` is
returned and this format type is not used.
"""
format_type = Unicode('text/plain')
_return_type = string_types
enabled = Bool(True, config=True)
print_method = ObjectName('__repr__')
# The singleton printers.
# Maps the IDs of the builtin singleton objects to the format functions.
singleton_printers = Dict(config=True)
# The type-specific printers.
# Map type objects to the format functions.
type_printers = Dict(config=True)
# The deferred-import type-specific printers.
# Map (modulename, classname) pairs to the format functions.
deferred_printers = Dict(config=True)
@warn_format_error
def __call__(self, obj):
"""Compute the format for an object."""
if self.enabled:
# lookup registered printer
try:
printer = self.lookup(obj)
except KeyError:
pass
else:
return printer(obj)
# Finally look for special method names
method = _safe_get_formatter_method(obj, self.print_method)
if method is not None:
return method()
return None
else:
return None
def __contains__(self, typ):
"""map in to lookup_by_type"""
try:
self.lookup_by_type(typ)
except KeyError:
return False
else:
return True
def lookup(self, obj):
"""Look up the formatter for a given instance.
Parameters
----------
obj : object instance
Returns
-------
f : callable
The registered formatting callable for the type.
Raises
------
KeyError if the type has not been registered.
"""
# look for singleton first
obj_id = id(obj)
if obj_id in self.singleton_printers:
return self.singleton_printers[obj_id]
# then lookup by type
return self.lookup_by_type(_get_type(obj))
def lookup_by_type(self, typ):
"""Look up the registered formatter for a type.
Parameters
----------
typ : type or '__module__.__name__' string for a type
Returns
-------
f : callable
The registered formatting callable for the type.
Raises
------
KeyError if the type has not been registered.
"""
if isinstance(typ, string_types):
typ_key = tuple(typ.rsplit('.', 1))
if typ_key not in self.deferred_printers:
# We may have it cached in the type map. We will have to
# iterate over all of the types to check.
for cls in self.type_printers:
if _mod_name_key(cls) == typ_key:
return self.type_printers[cls]
else:
return self.deferred_printers[typ_key]
else:
for cls in pretty._get_mro(typ):
if cls in self.type_printers or self._in_deferred_types(cls):
return self.type_printers[cls]
# If we have reached here, the lookup failed.
raise KeyError("No registered printer for {0!r}".format(typ))
def for_type(self, typ, func=None):
"""Add a format function for a given type.
Parameters
-----------
typ : type or '__module__.__name__' string for a type
The class of the object that will be formatted using `func`.
func : callable
A callable for computing the format data.
`func` will be called with the object to be formatted,
and will return the raw data in this formatter's format.
Subclasses may use a different call signature for the
`func` argument.
If `func` is None or not specified, there will be no change,
only returning the current value.
Returns
-------
oldfunc : callable
The currently registered callable.
If you are registering a new formatter,
this will be the previous value (to enable restoring later).
"""
# if string given, interpret as 'pkg.module.class_name'
if isinstance(typ, string_types):
type_module, type_name = typ.rsplit('.', 1)
return self.for_type_by_name(type_module, type_name, func)
try:
oldfunc = self.lookup_by_type(typ)
except KeyError:
oldfunc = None
if func is not None:
self.type_printers[typ] = func
return oldfunc
def for_type_by_name(self, type_module, type_name, func=None):
"""Add a format function for a type specified by the full dotted
module and name of the type, rather than the type of the object.
Parameters
----------
type_module : str
The full dotted name of the module the type is defined in, like
``numpy``.
type_name : str
The name of the type (the class name), like ``dtype``
func : callable
A callable for computing the format data.
`func` will be called with the object to be formatted,
and will return the raw data in this formatter's format.
Subclasses may use a different call signature for the
`func` argument.
If `func` is None or unspecified, there will be no change,
only returning the current value.
Returns
-------
oldfunc : callable
The currently registered callable.
If you are registering a new formatter,
this will be the previous value (to enable restoring later).
"""
key = (type_module, type_name)
try:
oldfunc = self.lookup_by_type("%s.%s" % key)
except KeyError:
oldfunc = None
if func is not None:
self.deferred_printers[key] = func
return oldfunc
def pop(self, typ, default=_raise_key_error):
"""Pop a formatter for the given type.
Parameters
----------
typ : type or '__module__.__name__' string for a type
default : object
value to be returned if no formatter is registered for typ.
Returns
-------
obj : object
The last registered object for the type.
Raises
------
KeyError if the type is not registered and default is not specified.
"""
if isinstance(typ, string_types):
typ_key = tuple(typ.rsplit('.', 1))
if typ_key not in self.deferred_printers:
# We may have it cached in the type map. We will have to
# iterate over all of the types to check.
for cls in self.type_printers:
if _mod_name_key(cls) == typ_key:
old = self.type_printers.pop(cls)
break
else:
old = default
else:
old = self.deferred_printers.pop(typ_key)
else:
if typ in self.type_printers:
old = self.type_printers.pop(typ)
else:
old = self.deferred_printers.pop(_mod_name_key(typ), default)
if old is _raise_key_error:
raise KeyError("No registered value for {0!r}".format(typ))
return old
def _in_deferred_types(self, cls):
"""
Check if the given class is specified in the deferred type registry.
Successful matches will be moved to the regular type registry for future use.
"""
mod = getattr(cls, '__module__', None)
name = getattr(cls, '__name__', None)
key = (mod, name)
if key in self.deferred_printers:
# Move the printer over to the regular registry.
printer = self.deferred_printers.pop(key)
self.type_printers[cls] = printer
return True
return False
class ProfileCreate(BaseIPythonApplication):
name = u'ipython-profile'
description = create_help
examples = _create_examples
auto_create = Bool(True, config=False)
def _copy_config_files_default(self):
return True
parallel = Bool(False,
config=True,
help="whether to include parallel computing config files")
def _parallel_changed(self, name, old, new):
parallel_files = [
'ipcontroller_config.py', 'ipengine_config.py',
'ipcluster_config.py'
]
if new:
for cf in parallel_files:
self.config_files.append(cf)
else:
for cf in parallel_files:
if cf in self.config_files:
self.config_files.remove(cf)
def parse_command_line(self, argv):
super(ProfileCreate, self).parse_command_line(argv)
# accept positional arg as profile name
if self.extra_args:
self.profile = self.extra_args[0]
flags = Dict(create_flags)
classes = [ProfileDir]
def init_config_files(self):
super(ProfileCreate, self).init_config_files()
# use local imports, since these classes may import from here
from IPython.terminal.ipapp import TerminalIPythonApp
apps = [TerminalIPythonApp]
try:
from IPython.qt.console.qtconsoleapp import IPythonQtConsoleApp
except Exception:
# this should be ImportError, but under weird circumstances
# this might be an AttributeError, or possibly others
# in any case, nothing should cause the profile creation to crash.
pass
else:
apps.append(IPythonQtConsoleApp)
try:
from IPython.html.notebookapp import NotebookApp
except ImportError:
pass
except Exception:
self.log.debug('Unexpected error when importing NotebookApp',
exc_info=True)
else:
apps.append(NotebookApp)
if self.parallel:
from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
from IPython.parallel.apps.ipengineapp import IPEngineApp
from IPython.parallel.apps.ipclusterapp import IPClusterStart
from IPython.parallel.apps.iploggerapp import IPLoggerApp
apps.extend([
IPControllerApp,
IPEngineApp,
IPClusterStart,
IPLoggerApp,
])
for App in apps:
app = App()
app.config.update(self.config)
app.log = self.log
app.overwrite = self.overwrite
app.copy_config_files = True
app.profile = self.profile
app.init_profile_dir()
app.init_config_files()
def stage_default_config_file(self):
pass
class PlainTextFormatter(BaseFormatter):
"""The default pretty-printer.
This uses :mod:`IPython.lib.pretty` to compute the format data of
the object. If the object cannot be pretty printed, :func:`repr` is used.
See the documentation of :mod:`IPython.lib.pretty` for details on
how to write pretty printers. Here is a simple example::
def dtype_pprinter(obj, p, cycle):
if cycle:
return p.text('dtype(...)')
if hasattr(obj, 'fields'):
if obj.fields is None:
p.text(repr(obj))
else:
p.begin_group(7, 'dtype([')
for i, field in enumerate(obj.descr):
if i > 0:
p.text(',')
p.breakable()
p.pretty(field)
p.end_group(7, '])')
"""
# The format type of data returned.
format_type = Unicode('text/plain')
# This subclass ignores this attribute as it always need to return
# something.
enabled = Bool(True, config=False)
# Look for a _repr_pretty_ methods to use for pretty printing.
print_method = ObjectName('_repr_pretty_')
# Whether to pretty-print or not.
pprint = Bool(True, config=True)
# Whether to be verbose or not.
verbose = Bool(False, config=True)
# The maximum width.
max_width = Integer(79, config=True)
# The newline character.
newline = Unicode('\n', config=True)
# format-string for pprinting floats
float_format = Unicode('%r')
# setter for float precision, either int or direct format-string
float_precision = CUnicode('', config=True)
def _float_precision_changed(self, name, old, new):
"""float_precision changed, set float_format accordingly.
float_precision can be set by int or str.
This will set float_format, after interpreting input.
If numpy has been imported, numpy print precision will also be set.
integer `n` sets format to '%.nf', otherwise, format set directly.
An empty string returns to defaults (repr for float, 8 for numpy).
This parameter can be set via the '%precision' magic.
"""
if '%' in new:
# got explicit format string
fmt = new
try:
fmt % 3.14159
except Exception:
raise ValueError(
"Precision must be int or format string, not %r" % new)
elif new:
# otherwise, should be an int
try:
i = int(new)
assert i >= 0
except ValueError:
raise ValueError(
"Precision must be int or format string, not %r" % new)
except AssertionError:
raise ValueError("int precision must be non-negative, not %r" %
i)
fmt = '%%.%if' % i
if 'numpy' in sys.modules:
# set numpy precision if it has been imported
import numpy
numpy.set_printoptions(precision=i)
else:
# default back to repr
fmt = '%r'
if 'numpy' in sys.modules:
import numpy
# numpy default is 8
numpy.set_printoptions(precision=8)
self.float_format = fmt
# Use the default pretty printers from IPython.lib.pretty.
def _singleton_printers_default(self):
return pretty._singleton_pprinters.copy()
def _type_printers_default(self):
d = pretty._type_pprinters.copy()
d[float] = lambda obj, p, cycle: p.text(self.float_format % obj)
return d
def _deferred_printers_default(self):
return pretty._deferred_type_pprinters.copy()
#### FormatterABC interface ####
@warn_format_error
def __call__(self, obj):
"""Compute the pretty representation of the object."""
if not self.pprint:
return pretty._safe_repr(obj)
else:
# This uses use StringIO, as cStringIO doesn't handle unicode.
stream = StringIO()
# self.newline.encode() is a quick fix for issue gh-597. We need to
# ensure that stream does not get a mix of unicode and bytestrings,
# or it will cause trouble.
printer = pretty.RepresentationPrinter(
stream,
self.verbose,
self.max_width,
unicode_to_str(self.newline),
singleton_pprinters=self.singleton_printers,
type_pprinters=self.type_printers,
deferred_pprinters=self.deferred_printers)
printer.pretty(obj)
printer.flush()
return stream.getvalue()
class FileNotebookManager(NotebookManager):
save_script = Bool(
False,
config=True,
help="""Automatically create a Python script when saving the notebook.
For easier use of import, %run and %load across notebooks, a
<notebook-name>.py script will be created next to any
<notebook-name>.ipynb on each save. This can also be set with the
short `--script` flag.
""")
checkpoint_dir = Unicode(
config=True,
help="""The location in which to keep notebook checkpoints
By default, it is notebook-dir/.ipynb_checkpoints
""")
def _checkpoint_dir_default(self):
return os.path.join(self.notebook_dir, '.ipynb_checkpoints')
def _checkpoint_dir_changed(self, name, old, new):
"""do a bit of validation of the checkpoint dir"""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
abs_new = os.path.abspath(new)
self.checkpoint_dir = abs_new
return
if os.path.exists(new) and not os.path.isdir(new):
raise TraitError("checkpoint dir %r is not a directory" % new)
if not os.path.exists(new):
self.log.info("Creating checkpoint dir %s", new)
try:
os.mkdir(new)
except:
raise TraitError("Couldn't create checkpoint dir %r" % new)
def get_notebook_names(self, path=''):
"""List all notebook names in the notebook dir and path."""
path = path.strip('/')
if not os.path.isdir(self.get_os_path(path=path)):
raise web.HTTPError(404, 'Directory not found: ' + path)
names = glob.glob(self.get_os_path('*' + self.filename_ext, path))
names = [os.path.basename(name) for name in names]
return names
def increment_filename(self, basename, path='', ext='.ipynb'):
"""Return a non-used filename of the form basename<int>."""
path = path.strip('/')
for i in itertools.count():
name = u'{basename}{i}{ext}'.format(basename=basename,
i=i,
ext=ext)
os_path = self.get_os_path(name, path)
if not os.path.isfile(os_path):
break
return name
def path_exists(self, path):
"""Does the API-style path (directory) actually exist?
Parameters
----------
path : string
The path to check. This is an API path (`/` separated,
relative to base notebook-dir).
Returns
-------
exists : bool
Whether the path is indeed a directory.
"""
path = path.strip('/')
os_path = self.get_os_path(path=path)
return os.path.isdir(os_path)
def get_os_path(self, name=None, path=''):
"""Given a notebook name and a URL path, return its file system
path.
Parameters
----------
name : string
The name of a notebook file with the .ipynb extension
path : string
The relative URL path (with '/' as separator) to the named
notebook.
Returns
-------
path : string
A file system path that combines notebook_dir (location where
server started), the relative path, and the filename with the
current operating system's url.
"""
parts = path.strip('/').split('/')
parts = [p for p in parts if p != ''] # remove duplicate splits
if name is not None:
parts.append(name)
path = os.path.join(self.notebook_dir, *parts)
return path
def notebook_exists(self, name, path=''):
"""Returns a True if the notebook exists. Else, returns False.
Parameters
----------
name : string
The name of the notebook you are checking.
path : string
The relative path to the notebook (with '/' as separator)
Returns
-------
bool
"""
path = path.strip('/')
nbpath = self.get_os_path(name, path=path)
return os.path.isfile(nbpath)
def list_notebooks(self, path):
"""Returns a list of dictionaries that are the standard model
for all notebooks in the relative 'path'.
Parameters
----------
path : str
the URL path that describes the relative path for the
listed notebooks
Returns
-------
notebooks : list of dicts
a list of the notebook models without 'content'
"""
path = path.strip('/')
notebook_names = self.get_notebook_names(path)
notebooks = []
for name in notebook_names:
model = self.get_notebook_model(name, path, content=False)
notebooks.append(model)
notebooks = sorted(notebooks, key=lambda item: item['name'])
return notebooks
def get_notebook_model(self, name, path='', content=True):
""" Takes a path and name for a notebook and returns it's model
Parameters
----------
name : str
the name of the notebook
path : str
the URL path that describes the relative path for
the notebook
Returns
-------
model : dict
the notebook model. If contents=True, returns the 'contents'
dict in the model as well.
"""
path = path.strip('/')
if not self.notebook_exists(name=name, path=path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % name)
os_path = self.get_os_path(name, path)
info = os.stat(os_path)
last_modified = tz.utcfromtimestamp(info.st_mtime)
created = tz.utcfromtimestamp(info.st_ctime)
# Create the notebook model.
model = {}
model['name'] = name
model['path'] = path
model['last_modified'] = last_modified
model['created'] = created
if content is True:
with io.open(os_path, 'r', encoding='utf-8') as f:
try:
nb = current.read(f, u'json')
except Exception as e:
raise web.HTTPError(
400, u"Unreadable Notebook: %s %s" % (os_path, e))
model['content'] = nb
return model
def save_notebook_model(self, model, name='', path=''):
"""Save the notebook model and return the model with no content."""
path = path.strip('/')
if 'content' not in model:
raise web.HTTPError(400, u'No notebook JSON data provided')
# One checkpoint should always exist
if self.notebook_exists(
name, path) and not self.list_checkpoints(name, path):
self.create_checkpoint(name, path)
new_path = model.get('path', path).strip('/')
new_name = model.get('name', name)
if path != new_path or name != new_name:
self.rename_notebook(name, path, new_name, new_path)
# Save the notebook file
os_path = self.get_os_path(new_name, new_path)
nb = current.to_notebook_json(model['content'])
if 'name' in nb['metadata']:
nb['metadata']['name'] = u''
try:
self.log.debug("Autosaving notebook %s", os_path)
with io.open(os_path, 'w', encoding='utf-8') as f:
current.write(nb, f, u'json')
except Exception as e:
raise web.HTTPError(
400, u'Unexpected error while autosaving notebook: %s %s' %
(os_path, e))
# Save .py script as well
if self.save_script:
py_path = os.path.splitext(os_path)[0] + '.py'
self.log.debug("Writing script %s", py_path)
try:
with io.open(py_path, 'w', encoding='utf-8') as f:
current.write(nb, f, u'py')
except Exception as e:
raise web.HTTPError(
400,
u'Unexpected error while saving notebook as script: %s %s'
% (py_path, e))
model = self.get_notebook_model(new_name, new_path, content=False)
return model
def update_notebook_model(self, model, name, path=''):
"""Update the notebook's path and/or name"""
path = path.strip('/')
new_name = model.get('name', name)
new_path = model.get('path', path).strip('/')
if path != new_path or name != new_name:
self.rename_notebook(name, path, new_name, new_path)
model = self.get_notebook_model(new_name, new_path, content=False)
return model
def delete_notebook_model(self, name, path=''):
"""Delete notebook by name and path."""
path = path.strip('/')
os_path = self.get_os_path(name, path)
if not os.path.isfile(os_path):
raise web.HTTPError(404, u'Notebook does not exist: %s' % os_path)
# clear checkpoints
for checkpoint in self.list_checkpoints(name, path):
checkpoint_id = checkpoint['id']
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if os.path.isfile(cp_path):
self.log.debug("Unlinking checkpoint %s", cp_path)
os.unlink(cp_path)
self.log.debug("Unlinking notebook %s", os_path)
os.unlink(os_path)
def rename_notebook(self, old_name, old_path, new_name, new_path):
"""Rename a notebook."""
old_path = old_path.strip('/')
new_path = new_path.strip('/')
if new_name == old_name and new_path == old_path:
return
new_os_path = self.get_os_path(new_name, new_path)
old_os_path = self.get_os_path(old_name, old_path)
# Should we proceed with the move?
if os.path.isfile(new_os_path):
raise web.HTTPError(
409, u'Notebook with name already exists: %s' % new_os_path)
if self.save_script:
old_py_path = os.path.splitext(old_os_path)[0] + '.py'
new_py_path = os.path.splitext(new_os_path)[0] + '.py'
if os.path.isfile(new_py_path):
raise web.HTTPError(
409, u'Python script with name already exists: %s' %
new_py_path)
# Move the notebook file
try:
os.rename(old_os_path, new_os_path)
except Exception as e:
raise web.HTTPError(
500,
u'Unknown error renaming notebook: %s %s' % (old_os_path, e))
# Move the checkpoints
old_checkpoints = self.list_checkpoints(old_name, old_path)
for cp in old_checkpoints:
checkpoint_id = cp['id']
old_cp_path = self.get_checkpoint_path(checkpoint_id, old_name,
old_path)
new_cp_path = self.get_checkpoint_path(checkpoint_id, new_name,
new_path)
if os.path.isfile(old_cp_path):
self.log.debug("Renaming checkpoint %s -> %s", old_cp_path,
new_cp_path)
os.rename(old_cp_path, new_cp_path)
# Move the .py script
if self.save_script:
os.rename(old_py_path, new_py_path)
# Checkpoint-related utilities
def get_checkpoint_path(self, checkpoint_id, name, path=''):
"""find the path to a checkpoint"""
path = path.strip('/')
basename, _ = os.path.splitext(name)
filename = u"{name}-{checkpoint_id}{ext}".format(
name=basename,
checkpoint_id=checkpoint_id,
ext=self.filename_ext,
)
cp_path = os.path.join(path, self.checkpoint_dir, filename)
return cp_path
def get_checkpoint_model(self, checkpoint_id, name, path=''):
"""construct the info dict for a given checkpoint"""
path = path.strip('/')
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
stats = os.stat(cp_path)
last_modified = tz.utcfromtimestamp(stats.st_mtime)
info = dict(
id=checkpoint_id,
last_modified=last_modified,
)
return info
# public checkpoint API
def create_checkpoint(self, name, path=''):
"""Create a checkpoint from the current state of a notebook"""
path = path.strip('/')
nb_path = self.get_os_path(name, path)
# only the one checkpoint ID:
checkpoint_id = u"checkpoint"
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
self.log.debug("creating checkpoint for notebook %s", name)
if not os.path.exists(self.checkpoint_dir):
os.mkdir(self.checkpoint_dir)
shutil.copy2(nb_path, cp_path)
# return the checkpoint info
return self.get_checkpoint_model(checkpoint_id, name, path)
def list_checkpoints(self, name, path=''):
"""list the checkpoints for a given notebook
This notebook manager currently only supports one checkpoint per notebook.
"""
path = path.strip('/')
checkpoint_id = "checkpoint"
path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.exists(path):
return []
else:
return [self.get_checkpoint_model(checkpoint_id, name, path)]
def restore_checkpoint(self, checkpoint_id, name, path=''):
"""restore a notebook to a checkpointed state"""
path = path.strip('/')
self.log.info("restoring Notebook %s from checkpoint %s", name,
checkpoint_id)
nb_path = self.get_os_path(name, path)
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.isfile(cp_path):
self.log.debug("checkpoint file does not exist: %s", cp_path)
raise web.HTTPError(
404, u'Notebook checkpoint does not exist: %s-%s' %
(name, checkpoint_id))
# ensure notebook is readable (never restore from an unreadable notebook)
with io.open(cp_path, 'r', encoding='utf-8') as f:
nb = current.read(f, u'json')
shutil.copy2(cp_path, nb_path)
self.log.debug("copying %s -> %s", cp_path, nb_path)
def delete_checkpoint(self, checkpoint_id, name, path=''):
"""delete a notebook's checkpoint"""
path = path.strip('/')
cp_path = self.get_checkpoint_path(checkpoint_id, name, path)
if not os.path.isfile(cp_path):
raise web.HTTPError(
404, u'Notebook checkpoint does not exist: %s%s-%s' %
(path, name, checkpoint_id))
self.log.debug("unlinking %s", cp_path)
os.unlink(cp_path)
def info_string(self):
return "Serving notebooks from local directory: %s" % self.notebook_dir
class IPKernelApp(BaseIPythonApplication, InteractiveShellApp):
name='ipkernel'
aliases = Dict(kernel_aliases)
flags = Dict(kernel_flags)
classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
# the kernel class, as an importstring
kernel_class = DottedObjectName('IPython.kernel.zmq.ipkernel.Kernel', config=True,
help="""The Kernel subclass to be used.
This should allow easy re-use of the IPKernelApp entry point
to configure and launch kernels other than IPython's own.
""")
kernel = Any()
poller = Any() # don't restrict this even though current pollers are all Threads
heartbeat = Instance(Heartbeat)
session = Instance('IPython.kernel.zmq.session.Session')
ports = Dict()
# inherit config file name from parent:
parent_appname = Unicode(config=True)
def _parent_appname_changed(self, name, old, new):
if self.config_file_specified:
# it was manually specified, ignore
return
self.config_file_name = new.replace('-','_') + u'_config.py'
# don't let this count as specifying the config file
self.config_file_specified = False
# connection info:
transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True)
ip = Unicode(config=True,
help="Set the IP or interface on which the kernel will listen.")
def _ip_default(self):
if self.transport == 'ipc':
if self.connection_file:
return os.path.splitext(self.abs_connection_file)[0] + '-ipc'
else:
return 'kernel-ipc'
else:
return LOCALHOST
hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]")
shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]")
iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]")
stdin_port = Integer(0, config=True, help="set the stdin (DEALER) port [default: random]")
connection_file = Unicode('', config=True,
help="""JSON file in which to store connection info [default: kernel-<pid>.json]
This file will contain the IP, ports, and authentication key needed to connect
clients to this kernel. By default, this file will be created in the security dir
of the current profile, but can be specified by absolute path.
""")
@property
def abs_connection_file(self):
if os.path.basename(self.connection_file) == self.connection_file:
return os.path.join(self.profile_dir.security_dir, self.connection_file)
else:
return self.connection_file
# streams, etc.
no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
outstream_class = DottedObjectName('IPython.kernel.zmq.iostream.OutStream',
config=True, help="The importstring for the OutStream factory")
displayhook_class = DottedObjectName('IPython.kernel.zmq.displayhook.ZMQDisplayHook',
config=True, help="The importstring for the DisplayHook factory")
# polling
parent = Integer(0, config=True,
help="""kill this process if its parent dies. On Windows, the argument
specifies the HANDLE of the parent process, otherwise it is simply boolean.
""")
interrupt = Integer(0, config=True,
help="""ONLY USED ON WINDOWS
Interrupt this process when the parent is signaled.
""")
def init_crash_handler(self):
# Install minimal exception handling
sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
ostream=sys.__stdout__)
def init_poller(self):
if sys.platform == 'win32':
if self.interrupt or self.parent:
self.poller = ParentPollerWindows(self.interrupt, self.parent)
elif self.parent:
self.poller = ParentPollerUnix()
def _bind_socket(self, s, port):
iface = '%s://%s' % (self.transport, self.ip)
if self.transport == 'tcp':
if port <= 0:
port = s.bind_to_random_port(iface)
else:
s.bind("tcp://%s:%i" % (self.ip, port))
elif self.transport == 'ipc':
if port <= 0:
port = 1
path = "%s-%i" % (self.ip, port)
while os.path.exists(path):
port = port + 1
path = "%s-%i" % (self.ip, port)
else:
path = "%s-%i" % (self.ip, port)
s.bind("ipc://%s" % path)
return port
def load_connection_file(self):
"""load ip/port/hmac config from JSON connection file"""
try:
fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
except IOError:
self.log.debug("Connection file not found: %s", self.connection_file)
# This means I own it, so I will clean it up:
atexit.register(self.cleanup_connection_file)
return
self.log.debug(u"Loading connection file %s", fname)
with open(fname) as f:
s = f.read()
cfg = json.loads(s)
self.transport = cfg.get('transport', self.transport)
if self.ip == self._ip_default() and 'ip' in cfg:
# not overridden by config or cl_args
self.ip = cfg['ip']
for channel in ('hb', 'shell', 'iopub', 'stdin'):
name = channel + '_port'
if getattr(self, name) == 0 and name in cfg:
# not overridden by config or cl_args
setattr(self, name, cfg[name])
if 'key' in cfg:
self.config.Session.key = str_to_bytes(cfg['key'])
def write_connection_file(self):
"""write connection info to JSON file"""
cf = self.abs_connection_file
self.log.debug("Writing connection file: %s", cf)
write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport,
shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
iopub_port=self.iopub_port)
def cleanup_connection_file(self):
cf = self.abs_connection_file
self.log.debug("Cleaning up connection file: %s", cf)
try:
os.remove(cf)
except (IOError, OSError):
pass
self.cleanup_ipc_files()
def cleanup_ipc_files(self):
"""cleanup ipc files if we wrote them"""
if self.transport != 'ipc':
return
for port in (self.shell_port, self.iopub_port, self.stdin_port, self.hb_port):
ipcfile = "%s-%i" % (self.ip, port)
try:
os.remove(ipcfile)
except (IOError, OSError):
pass
def init_connection_file(self):
if not self.connection_file:
self.connection_file = "kernel-%s.json"%os.getpid()
try:
self.load_connection_file()
except Exception:
self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
self.exit(1)
def init_sockets(self):
# Create a context, a session, and the kernel sockets.
self.log.info("Starting the kernel at pid: %i", os.getpid())
context = zmq.Context.instance()
# Uncomment this to try closing the context.
# atexit.register(context.term)
self.shell_socket = context.socket(zmq.ROUTER)
self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
self.log.debug("shell ROUTER Channel on port: %i"%self.shell_port)
self.iopub_socket = context.socket(zmq.PUB)
self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
self.log.debug("iopub PUB Channel on port: %i"%self.iopub_port)
self.stdin_socket = context.socket(zmq.ROUTER)
self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
self.log.debug("stdin ROUTER Channel on port: %i"%self.stdin_port)
def init_heartbeat(self):
"""start the heart beating"""
# heartbeat doesn't share context, because it mustn't be blocked
# by the GIL, which is accessed by libzmq when freeing zero-copy messages
hb_ctx = zmq.Context()
self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port))
self.hb_port = self.heartbeat.port
self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
self.heartbeat.start()
# Helper to make it easier to connect to an existing kernel.
# set log-level to critical, to make sure it is output
self.log.critical("To connect another client to this kernel, use:")
def log_connection_info(self):
"""display connection info, and store ports"""
basename = os.path.basename(self.connection_file)
if basename == self.connection_file or \
os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
# use shortname
tail = basename
if self.profile != 'default':
tail += " --profile %s" % self.profile
else:
tail = self.connection_file
self.log.critical("--existing %s", tail)
self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
stdin=self.stdin_port, hb=self.hb_port)
def init_session(self):
"""create our session object"""
default_secure(self.config)
self.session = Session(config=self.config, username=u'kernel')
def init_blackhole(self):
"""redirects stdout/stderr to devnull if necessary"""
if self.no_stdout or self.no_stderr:
blackhole = open(os.devnull, 'w')
if self.no_stdout:
sys.stdout = sys.__stdout__ = blackhole
if self.no_stderr:
sys.stderr = sys.__stderr__ = blackhole
def init_io(self):
"""Redirect input streams and set a display hook."""
if self.outstream_class:
outstream_factory = import_item(str(self.outstream_class))
sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
if self.displayhook_class:
displayhook_factory = import_item(str(self.displayhook_class))
sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
def init_signal(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
def init_kernel(self):
"""Create the Kernel object itself"""
shell_stream = ZMQStream(self.shell_socket)
kernel_factory = import_item(str(self.kernel_class))
kernel = kernel_factory(config=self.config, session=self.session,
shell_streams=[shell_stream],
iopub_socket=self.iopub_socket,
stdin_socket=self.stdin_socket,
log=self.log,
profile_dir=self.profile_dir,
)
kernel.record_ports(self.ports)
self.kernel = kernel
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
# Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab`
# to ensure that any exception is printed straight to stderr.
# Normally _showtraceback associates the reply with an execution,
# which means frontends will never draw it, as this exception
# is not associated with any execute request.
shell = self.shell
_showtraceback = shell._showtraceback
try:
# replace pyerr-sending traceback with stderr
def print_tb(etype, evalue, stb):
print ("GUI event loop or pylab initialization failed",
file=io.stderr)
print (shell.InteractiveTB.stb2text(stb), file=io.stderr)
shell._showtraceback = print_tb
InteractiveShellApp.init_gui_pylab(self)
finally:
shell._showtraceback = _showtraceback
def init_shell(self):
self.shell = self.kernel.shell
self.shell.configurables.append(self)
@catch_config_error
def initialize(self, argv=None):
super(IPKernelApp, self).initialize(argv)
self.init_blackhole()
self.init_connection_file()
self.init_session()
self.init_poller()
self.init_sockets()
self.init_heartbeat()
# writing/displaying connection info must be *after* init_sockets/heartbeat
self.log_connection_info()
self.write_connection_file()
self.init_io()
self.init_signal()
self.init_kernel()
# shell init steps
self.init_path()
self.init_shell()
self.init_gui_pylab()
self.init_extensions()
self.init_code()
# flush stdout/stderr, so that anything written to these streams during
# initialization do not get associated with the first execution request
sys.stdout.flush()
sys.stderr.flush()
def start(self):
if self.poller is not None:
self.poller.start()
self.kernel.start()
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
pass
class Bar(Configurable):
b = Integer(0, config=True, help="The integer b.")
enabled = Bool(True, config=True, help="Enable bar.")
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):
""" A Qt frontend for a generic Python kernel.
"""
# The text to show when the kernel is (re)started.
banner = Unicode()
# An option and corresponding signal for overriding the default kernel
# interrupt behavior.
custom_interrupt = Bool(False)
custom_interrupt_requested = QtCore.Signal()
# An option and corresponding signals for overriding the default kernel
# restart behavior.
custom_restart = Bool(False)
custom_restart_kernel_died = QtCore.Signal(float)
custom_restart_requested = QtCore.Signal()
# Whether to automatically show calltips on open-parentheses.
enable_calltips = Bool(
True,
config=True,
help="Whether to draw information calltips on open-parentheses.")
clear_on_kernel_restart = Bool(
True,
config=True,
help="Whether to clear the console when the kernel is restarted")
confirm_restart = Bool(
True,
config=True,
help="Whether to ask for user confirmation when restarting kernel")
# Emitted when a user visible 'execute_request' has been submitted to the
# kernel from the FrontendWidget. Contains the code to be executed.
executing = QtCore.Signal(object)
# Emitted when a user-visible 'execute_reply' has been received from the
# kernel and processed by the FrontendWidget. Contains the response message.
executed = QtCore.Signal(object)
# Emitted when an exit request has been received from the kernel.
exit_requested = QtCore.Signal(object)
# Protected class variables.
_prompt_transformer = IPythonInputSplitter(
physical_line_transforms=[classic_prompt()],
logical_line_transforms=[],
python_line_transforms=[],
)
_CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos'])
_CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos'])
_ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
_input_splitter_class = InputSplitter
_local_kernel = False
_highlighter = Instance(FrontendHighlighter)
#---------------------------------------------------------------------------
# 'object' interface
#---------------------------------------------------------------------------
def __init__(self, *args, **kw):
super(FrontendWidget, self).__init__(*args, **kw)
# FIXME: remove this when PySide min version is updated past 1.0.7
# forcefully disable calltips if PySide is < 1.0.7, because they crash
if qt.QT_API == qt.QT_API_PYSIDE:
import PySide
if PySide.__version_info__ < (1, 0, 7):
self.log.warn(
"PySide %s < 1.0.7 detected, disabling calltips" %
PySide.__version__)
self.enable_calltips = False
# FrontendWidget protected variables.
self._bracket_matcher = BracketMatcher(self._control)
self._call_tip_widget = CallTipWidget(self._control)
self._completion_lexer = CompletionLexer(PythonLexer())
self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
self._hidden = False
self._highlighter = FrontendHighlighter(self)
self._input_splitter = self._input_splitter_class(input_mode='cell')
self._kernel_manager = None
self._request_info = {}
self._request_info['execute'] = {}
self._callback_dict = {}
# Configure the ConsoleWidget.
self.tab_width = 4
self._set_continuation_prompt('... ')
# Configure the CallTipWidget.
self._call_tip_widget.setFont(self.font)
self.font_changed.connect(self._call_tip_widget.setFont)
# Configure actions.
action = self._copy_raw_action
key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C
action.setEnabled(False)
action.setShortcut(QtGui.QKeySequence(key))
action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut)
action.triggered.connect(self.copy_raw)
self.copy_available.connect(action.setEnabled)
self.addAction(action)
# Connect signal handlers.
document = self._control.document()
document.contentsChange.connect(self._document_contents_change)
# Set flag for whether we are connected via localhost.
self._local_kernel = kw.get('local_kernel',
FrontendWidget._local_kernel)
#---------------------------------------------------------------------------
# 'ConsoleWidget' public interface
#---------------------------------------------------------------------------
def copy(self):
""" Copy the currently selected text to the clipboard, removing prompts.
"""
if self._page_control is not None and self._page_control.hasFocus():
self._page_control.copy()
elif self._control.hasFocus():
text = self._control.textCursor().selection().toPlainText()
if text:
text = self._prompt_transformer.transform_cell(text)
QtGui.QApplication.clipboard().setText(text)
else:
self.log.debug("frontend widget : unknown copy target")
#---------------------------------------------------------------------------
# 'ConsoleWidget' abstract interface
#---------------------------------------------------------------------------
def _is_complete(self, source, interactive):
""" Returns whether 'source' can be completely processed and a new
prompt created. When triggered by an Enter/Return key press,
'interactive' is True; otherwise, it is False.
"""
complete = self._input_splitter.push(source)
if interactive:
complete = not self._input_splitter.push_accepts_more()
return complete
def _execute(self, source, hidden):
""" Execute 'source'. If 'hidden', do not show any output.
See parent class :meth:`execute` docstring for full details.
"""
msg_id = self.kernel_manager.shell_channel.execute(source, hidden)
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'user')
self._hidden = hidden
if not hidden:
self.executing.emit(source)
def _prompt_started_hook(self):
""" Called immediately after a new prompt is displayed.
"""
if not self._reading:
self._highlighter.highlighting_on = True
def _prompt_finished_hook(self):
""" Called immediately after a prompt is finished, i.e. when some input
will be processed and a new prompt displayed.
"""
# Flush all state from the input splitter so the next round of
# reading input starts with a clean buffer.
self._input_splitter.reset()
if not self._reading:
self._highlighter.highlighting_on = False
def _tab_pressed(self):
""" Called when the tab key is pressed. Returns whether to continue
processing the event.
"""
# Perform tab completion if:
# 1) The cursor is in the input buffer.
# 2) There is a non-whitespace character before the cursor.
text = self._get_input_buffer_cursor_line()
if text is None:
return False
complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
if complete:
self._complete()
return not complete
#---------------------------------------------------------------------------
# 'ConsoleWidget' protected interface
#---------------------------------------------------------------------------
def _context_menu_make(self, pos):
""" Reimplemented to add an action for raw copy.
"""
menu = super(FrontendWidget, self)._context_menu_make(pos)
for before_action in menu.actions():
if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \
QtGui.QKeySequence.ExactMatch:
menu.insertAction(before_action, self._copy_raw_action)
break
return menu
def request_interrupt_kernel(self):
if self._executing:
self.interrupt_kernel()
def request_restart_kernel(self):
message = 'Are you sure you want to restart the kernel?'
self.restart_kernel(message, now=False)
def _event_filter_console_keypress(self, event):
""" Reimplemented for execution interruption and smart backspace.
"""
key = event.key()
if self._control_key_down(event.modifiers(), include_command=False):
if key == QtCore.Qt.Key_C and self._executing:
self.request_interrupt_kernel()
return True
elif key == QtCore.Qt.Key_Period:
self.request_restart_kernel()
return True
elif not event.modifiers() & QtCore.Qt.AltModifier:
# Smart backspace: remove four characters in one backspace if:
# 1) everything left of the cursor is whitespace
# 2) the four characters immediately left of the cursor are spaces
if key == QtCore.Qt.Key_Backspace:
col = self._get_input_buffer_cursor_column()
cursor = self._control.textCursor()
if col > 3 and not cursor.hasSelection():
text = self._get_input_buffer_cursor_line()[:col]
if text.endswith(' ') and not text.strip():
cursor.movePosition(QtGui.QTextCursor.Left,
QtGui.QTextCursor.KeepAnchor, 4)
cursor.removeSelectedText()
return True
return super(FrontendWidget,
self)._event_filter_console_keypress(event)
def _insert_continuation_prompt(self, cursor):
""" Reimplemented for auto-indentation.
"""
super(FrontendWidget, self)._insert_continuation_prompt(cursor)
cursor.insertText(' ' * self._input_splitter.indent_spaces)
#---------------------------------------------------------------------------
# 'BaseFrontendMixin' abstract interface
#---------------------------------------------------------------------------
def _handle_complete_reply(self, rep):
""" Handle replies for tab completion.
"""
self.log.debug("complete: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('complete')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
text = '.'.join(self._get_context())
cursor.movePosition(QtGui.QTextCursor.Left, n=len(text))
self._complete_with_items(cursor, rep['content']['matches'])
def _silent_exec_callback(self, expr, callback):
"""Silently execute `expr` in the kernel and call `callback` with reply
the `expr` is evaluated silently in the kernel (without) output in
the frontend. Call `callback` with the
`repr <http://docs.python.org/library/functions.html#repr> `_ as first argument
Parameters
----------
expr : string
valid string to be executed by the kernel.
callback : function
function accepting one argument, as a string. The string will be
the `repr` of the result of evaluating `expr`
The `callback` is called with the `repr()` of the result of `expr` as
first argument. To get the object, do `eval()` on the passed value.
See Also
--------
_handle_exec_callback : private method, deal with calling callback with reply
"""
# generate uuid, which would be used as an indication of whether or
# not the unique request originated from here (can use msg id ?)
local_uuid = str(uuid.uuid1())
msg_id = self.kernel_manager.shell_channel.execute(
'', silent=True, user_expressions={local_uuid: expr})
self._callback_dict[local_uuid] = callback
self._request_info['execute'][msg_id] = self._ExecutionRequest(
msg_id, 'silent_exec_callback')
def _handle_exec_callback(self, msg):
"""Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``
Parameters
----------
msg : raw message send by the kernel containing an `user_expressions`
and having a 'silent_exec_callback' kind.
Notes
-----
This function will look for a `callback` associated with the
corresponding message id. Association has been made by
`_silent_exec_callback`. `callback` is then called with the `repr()`
of the value of corresponding `user_expressions` as argument.
`callback` is then removed from the known list so that any message
coming again with the same id won't trigger it.
"""
user_exp = msg['content'].get('user_expressions')
if not user_exp:
return
for expression in user_exp:
if expression in self._callback_dict:
self._callback_dict.pop(expression)(user_exp[expression])
def _handle_execute_reply(self, msg):
""" Handles replies for code execution.
"""
self.log.debug("execute: %s", msg.get('content', ''))
msg_id = msg['parent_header']['msg_id']
info = self._request_info['execute'].get(msg_id)
# unset reading flag, because if execute finished, raw_input can't
# still be pending.
self._reading = False
if info and info.kind == 'user' and not self._hidden:
# Make sure that all output from the SUB channel has been processed
# before writing a new prompt.
self.kernel_manager.iopub_channel.flush()
# Reset the ANSI style information to prevent bad text in stdout
# from messing up our colors. We're not a true terminal so we're
# allowed to do this.
if self.ansi_codes:
self._ansi_processor.reset_sgr()
content = msg['content']
status = content['status']
if status == 'ok':
self._process_execute_ok(msg)
elif status == 'error':
self._process_execute_error(msg)
elif status == 'aborted':
self._process_execute_abort(msg)
self._show_interpreter_prompt_for_reply(msg)
self.executed.emit(msg)
self._request_info['execute'].pop(msg_id)
elif info and info.kind == 'silent_exec_callback' and not self._hidden:
self._handle_exec_callback(msg)
self._request_info['execute'].pop(msg_id)
else:
super(FrontendWidget, self)._handle_execute_reply(msg)
def _handle_input_request(self, msg):
""" Handle requests for raw_input.
"""
self.log.debug("input: %s", msg.get('content', ''))
if self._hidden:
raise RuntimeError(
'Request for raw input during hidden execution.')
# Make sure that all output from the SUB channel has been processed
# before entering readline mode.
self.kernel_manager.iopub_channel.flush()
def callback(line):
self.kernel_manager.stdin_channel.input(line)
if self._reading:
self.log.debug(
"Got second input request, assuming first was interrupted.")
self._reading = False
self._readline(msg['content']['prompt'], callback=callback)
def _handle_kernel_died(self, since_last_heartbeat):
""" Handle the kernel's death by asking if the user wants to restart.
"""
self.log.debug("kernel died: %s", since_last_heartbeat)
if self.custom_restart:
self.custom_restart_kernel_died.emit(since_last_heartbeat)
else:
message = 'The kernel heartbeat has been inactive for %.2f ' \
'seconds. Do you want to restart the kernel? You may ' \
'first want to check the network connection.' % \
since_last_heartbeat
self.restart_kernel(message, now=True)
def _handle_object_info_reply(self, rep):
""" Handle replies for call tips.
"""
self.log.debug("oinfo: %s", rep.get('content', ''))
cursor = self._get_cursor()
info = self._request_info.get('call_tip')
if info and info.id == rep['parent_header']['msg_id'] and \
info.pos == cursor.position():
# Get the information for a call tip. For now we format the call
# line as string, later we can pass False to format_call and
# syntax-highlight it ourselves for nicer formatting in the
# calltip.
content = rep['content']
# if this is from pykernel, 'docstring' will be the only key
if content.get('ismagic', False):
# Don't generate a call-tip for magics. Ideally, we should
# generate a tooltip, but not on ( like we do for actual
# callables.
call_info, doc = None, None
else:
call_info, doc = call_tip(content, format_call=True)
if call_info or doc:
self._call_tip_widget.show_call_info(call_info, doc)
def _handle_pyout(self, msg):
""" Handle display hook output.
"""
self.log.debug("pyout: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
text = msg['content']['data']
self._append_plain_text(text + '\n', before_prompt=True)
def _handle_stream(self, msg):
""" Handle stdout, stderr, and stdin.
"""
self.log.debug("stream: %s", msg.get('content', ''))
if not self._hidden and self._is_from_this_session(msg):
# Most consoles treat tabs as being 8 space characters. Convert tabs
# to spaces so that output looks as expected regardless of this
# widget's tab width.
text = msg['content']['data'].expandtabs(8)
self._append_plain_text(text, before_prompt=True)
self._control.moveCursor(QtGui.QTextCursor.End)
def _handle_shutdown_reply(self, msg):
""" Handle shutdown signal, only if from other console.
"""
self.log.debug("shutdown: %s", msg.get('content', ''))
if not self._hidden and not self._is_from_this_session(msg):
if self._local_kernel:
if not msg['content']['restart']:
self.exit_requested.emit(self)
else:
# we just got notified of a restart!
time.sleep(0.25) # wait 1/4 sec to reset
# lest the request for a new prompt
# goes to the old kernel
self.reset()
else: # remote kernel, prompt on Kernel shutdown/reset
title = self.window().windowTitle()
if not msg['content']['restart']:
reply = QtGui.QMessageBox.question(
self, title, "Kernel has been shutdown permanently. "
"Close the Console?", QtGui.QMessageBox.Yes,
QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
self.exit_requested.emit(self)
else:
# XXX: remove message box in favor of using the
# clear_on_kernel_restart setting?
reply = QtGui.QMessageBox.question(
self, title,
"Kernel has been reset. Clear the Console?",
QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.Yes:
time.sleep(0.25) # wait 1/4 sec to reset
# lest the request for a new prompt
# goes to the old kernel
self.reset()
def _started_channels(self):
""" Called when the KernelManager channels have started listening or
when the frontend is assigned an already listening KernelManager.
"""
self.reset(clear=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' public interface
#---------------------------------------------------------------------------
def copy_raw(self):
""" Copy the currently selected text to the clipboard without attempting
to remove prompts or otherwise alter the text.
"""
self._control.copy()
def execute_file(self, path, hidden=False):
""" Attempts to execute file with 'path'. If 'hidden', no output is
shown.
"""
self.execute('execfile(%r)' % path, hidden=hidden)
def interrupt_kernel(self):
""" Attempts to interrupt the running kernel.
Also unsets _reading flag, to avoid runtime errors
if raw_input is called again.
"""
if self.custom_interrupt:
self._reading = False
self.custom_interrupt_requested.emit()
elif self.kernel_manager.has_kernel:
self._reading = False
self.kernel_manager.interrupt_kernel()
else:
self._append_plain_text('Kernel process is either remote or '
'unspecified. Cannot interrupt.\n')
def reset(self, clear=False):
""" Resets the widget to its initial state if ``clear`` parameter or
``clear_on_kernel_restart`` configuration setting is True, otherwise
prints a visual indication of the fact that the kernel restarted, but
does not clear the traces from previous usage of the kernel before it
was restarted. With ``clear=True``, it is similar to ``%clear``, but
also re-writes the banner and aborts execution if necessary.
"""
if self._executing:
self._executing = False
self._request_info['execute'] = {}
self._reading = False
self._highlighter.highlighting_on = False
if self.clear_on_kernel_restart or clear:
self._control.clear()
self._append_plain_text(self.banner)
else:
self._append_plain_text("# restarting kernel...")
self._append_html("<hr><br>")
# XXX: Reprinting the full banner may be too much, but once #1680 is
# addressed, that will mitigate it.
#self._append_plain_text(self.banner)
# update output marker for stdout/stderr, so that startup
# messages appear after banner:
self._append_before_prompt_pos = self._get_cursor().position()
self._show_interpreter_prompt()
def restart_kernel(self, message, now=False):
""" Attempts to restart the running kernel.
"""
# FIXME: now should be configurable via a checkbox in the dialog. Right
# now at least the heartbeat path sets it to True and the manual restart
# to False. But those should just be the pre-selected states of a
# checkbox that the user could override if so desired. But I don't know
# enough Qt to go implementing the checkbox now.
if self.custom_restart:
self.custom_restart_requested.emit()
elif self.kernel_manager.has_kernel:
# Pause the heart beat channel to prevent further warnings.
self.kernel_manager.hb_channel.pause()
# Prompt the user to restart the kernel. Un-pause the heartbeat if
# they decline. (If they accept, the heartbeat will be un-paused
# automatically when the kernel is restarted.)
if self.confirm_restart:
buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
result = QtGui.QMessageBox.question(self, 'Restart kernel?',
message, buttons)
do_restart = result == QtGui.QMessageBox.Yes
else:
# confirm_restart is False, so we don't need to ask user
# anything, just do the restart
do_restart = True
if do_restart:
try:
self.kernel_manager.restart_kernel(now=now)
except RuntimeError:
self._append_plain_text(
'Kernel started externally. '
'Cannot restart.\n',
before_prompt=True)
else:
self.reset()
else:
self.kernel_manager.hb_channel.unpause()
else:
self._append_plain_text(
'Kernel process is either remote or '
'unspecified. Cannot restart.\n',
before_prompt=True)
#---------------------------------------------------------------------------
# 'FrontendWidget' protected interface
#---------------------------------------------------------------------------
def _call_tip(self):
""" Shows a call tip, if appropriate, at the current cursor location.
"""
# Decide if it makes sense to show a call tip
if not self.enable_calltips:
return False
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.Left)
if cursor.document().characterAt(cursor.position()) != '(':
return False
context = self._get_context(cursor)
if not context:
return False
# Send the metadata request to the kernel
name = '.'.join(context)
msg_id = self.kernel_manager.shell_channel.object_info(name)
pos = self._get_cursor().position()
self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
return True
def _complete(self):
""" Performs completion at the current cursor location.
"""
context = self._get_context()
if context:
# Send the completion request to the kernel
msg_id = self.kernel_manager.shell_channel.complete(
'.'.join(context), # text
self._get_input_buffer_cursor_line(), # line
self._get_input_buffer_cursor_column(), # cursor_pos
self.input_buffer) # block
pos = self._get_cursor().position()
info = self._CompletionRequest(msg_id, pos)
self._request_info['complete'] = info
def _get_context(self, cursor=None):
""" Gets the context for the specified cursor (or the current cursor
if none is specified).
"""
if cursor is None:
cursor = self._get_cursor()
cursor.movePosition(QtGui.QTextCursor.StartOfBlock,
QtGui.QTextCursor.KeepAnchor)
text = cursor.selection().toPlainText()
return self._completion_lexer.get_context(text)
def _process_execute_abort(self, msg):
""" Process a reply for an aborted execution request.
"""
self._append_plain_text("ERROR: execution aborted\n")
def _process_execute_error(self, msg):
""" Process a reply for an execution request that resulted in an error.
"""
content = msg['content']
# If a SystemExit is passed along, this means exit() was called - also
# all the ipython %exit magic syntax of '-k' to be used to keep
# the kernel running
if content['ename'] == 'SystemExit':
keepkernel = content['evalue'] == '-k' or content[
'evalue'] == 'True'
self._keep_kernel_on_exit = keepkernel
self.exit_requested.emit(self)
else:
traceback = ''.join(content['traceback'])
self._append_plain_text(traceback)
def _process_execute_ok(self, msg):
""" Process a reply for a successful execution request.
"""
payload = msg['content']['payload']
for item in payload:
if not self._process_execute_payload(item):
warning = 'Warning: received unknown payload of type %s'
print(warning % repr(item['source']))
def _process_execute_payload(self, item):
""" Process a single payload item from the list of payload items in an
execution reply. Returns whether the payload was handled.
"""
# The basic FrontendWidget doesn't handle payloads, as they are a
# mechanism for going beyond the standard Python interpreter model.
return False
def _show_interpreter_prompt(self):
""" Shows a prompt for the interpreter.
"""
self._show_prompt('>>> ')
def _show_interpreter_prompt_for_reply(self, msg):
""" Shows a prompt for the interpreter given an 'execute_reply' message.
"""
self._show_interpreter_prompt()
#------ Signal handlers ----------------------------------------------------
def _document_contents_change(self, position, removed, added):
""" Called whenever the document's content changes. Display a call tip
if appropriate.
"""
# Calculate where the cursor should be *after* the change:
position += added
document = self._control.document()
if position == self._get_cursor().position():
self._call_tip()
#------ Trait default initializers -----------------------------------------
def _banner_default(self):
""" Returns the standard Python banner.
"""
banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
'"license" for more information.'
return banner % (sys.version, sys.platform)
class Comm(LoggingConfigurable):
# If this is instantiated by a non-IPython kernel, shell will be None
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
allow_none=True)
kernel = Instance('IPython.kernel.zmq.kernelbase.Kernel')
def _kernel_default(self):
if Kernel.initialized():
return Kernel.instance()
iopub_socket = Any()
def _iopub_socket_default(self):
return self.kernel.iopub_socket
session = Instance('IPython.kernel.zmq.session.Session')
def _session_default(self):
if self.kernel is not None:
return self.kernel.session
target_name = Unicode('comm')
topic = Bytes()
def _topic_default(self):
return ('comm-%s' % self.comm_id).encode('ascii')
_open_data = Dict(help="data dict, if any, to be included in comm_open")
_close_data = Dict(help="data dict, if any, to be included in comm_close")
_msg_callback = Any()
_close_callback = Any()
_closed = Bool(False)
comm_id = Unicode()
def _comm_id_default(self):
return uuid.uuid4().hex
primary = Bool(True, help="Am I the primary or secondary Comm?")
def __init__(self, target_name='', data=None, **kwargs):
if target_name:
kwargs['target_name'] = target_name
super(Comm, self).__init__(**kwargs)
if self.primary:
# I am primary, open my peer.
self.open(data)
def _publish_msg(self,
msg_type,
data=None,
metadata=None,
buffers=None,
**keys):
"""Helper for sending a comm message on IOPub"""
data = {} if data is None else data
metadata = {} if metadata is None else metadata
content = json_clean(dict(data=data, comm_id=self.comm_id, **keys))
self.session.send(
self.iopub_socket,
msg_type,
content,
metadata=json_clean(metadata),
parent=self.kernel._parent_header,
ident=self.topic,
buffers=buffers,
)
def __del__(self):
"""trigger close on gc"""
self.close()
# publishing messages
def open(self, data=None, metadata=None, buffers=None):
"""Open the frontend-side version of this comm"""
if data is None:
data = self._open_data
comm_manager = getattr(self.kernel, 'comm_manager', None)
if comm_manager is None:
raise RuntimeError("Comms cannot be opened without a kernel "
"and a comm_manager attached to that kernel.")
comm_manager.register_comm(self)
self._closed = False
self._publish_msg(
'comm_open',
data=data,
metadata=metadata,
buffers=buffers,
target_name=self.target_name,
)
def close(self, data=None, metadata=None, buffers=None):
"""Close the frontend-side version of this comm"""
if self._closed:
# only close once
return
if data is None:
data = self._close_data
self._publish_msg(
'comm_close',
data=data,
metadata=metadata,
buffers=buffers,
)
self.kernel.comm_manager.unregister_comm(self)
self._closed = True
def send(self, data=None, metadata=None, buffers=None):
"""Send a message to the frontend-side version of this comm"""
self._publish_msg(
'comm_msg',
data=data,
metadata=metadata,
buffers=buffers,
)
# registering callbacks
def on_close(self, callback):
"""Register a callback for comm_close
Will be called with the `data` of the close message.
Call `on_close(None)` to disable an existing callback.
"""
self._close_callback = callback
def on_msg(self, callback):
"""Register a callback for comm_msg
Will be called with the `data` of any comm_msg messages.
Call `on_msg(None)` to disable an existing callback.
"""
self._msg_callback = callback
# handling of incoming messages
def handle_close(self, msg):
"""Handle a comm_close message"""
self.log.debug("handle_close[%s](%s)", self.comm_id, msg)
if self._close_callback:
self._close_callback(msg)
def handle_msg(self, msg):
"""Handle a comm_msg message"""
self.log.debug("handle_msg[%s](%s)", self.comm_id, msg)
if self._msg_callback:
if self.shell:
self.shell.events.trigger('pre_execute')
self._msg_callback(msg)
if self.shell:
self.shell.events.trigger('post_execute')
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
default_config_file_name = 'ipython_notebook_config.py'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = IPythonConsoleApp.classes + [
MappingKernelManager, NotebookManager, FileNotebookManager
]
flags = Dict(flags)
aliases = Dict(aliases)
kernel_argv = List(Unicode)
log_level = Enum(
(0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'),
default_value=logging.INFO,
config=True,
help="Set the log level by value or name.")
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('')
# Network related information.
ip = Unicode(LOCALHOST,
config=True,
help="The IP address the notebook server will listen on.")
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888,
config=True,
help="The port the notebook server will listen on.")
port_retries = Integer(
50,
config=True,
help=
"The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(
u'',
config=True,
help="""The full path to an SSL/TLS certificate file.""")
keyfile = Unicode(
u'',
config=True,
help="""The full path to a private key file for usage with SSL/TLS.""")
password = Unicode(u'',
config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
""")
open_browser = Bool(True,
config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'',
config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
read_only = Bool(False,
config=True,
help="Whether to prevent editing/execution of notebooks.")
use_less = Bool(False,
config=True,
help="""Wether to use Browser Side less-css parsing
instead of compiled css version in templates that allows
it. This is mainly convenient when working on the less
file to avoid a build step, or if user want to overwrite
some of the less variables without having to recompile
everything.
You will need to install the less.js component in the static directory
either in the source tree or in your profile folder.
""")
webapp_settings = Dict(
config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
enable_mathjax = Bool(
True,
config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
""")
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_project_url = Unicode('/',
config=True,
help='''The base URL for the notebook server.
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_project_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_project_url = '/' + new
elif not new.endswith('/'):
self.base_project_url = new + '/'
base_kernel_url = Unicode('/',
config=True,
help='''The base URL for the kernel server
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_kernel_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_kernel_url = '/' + new
elif not new.endswith('/'):
self.base_kernel_url = new + '/'
websocket_host = Unicode("",
config=True,
help="""The hostname for the websocket server.""")
extra_static_paths = List(
Unicode,
config=True,
help="""Extra paths to search for serving static files.
This allows adding javascript/css to be available from the notebook server machine,
or overriding individual files in the IPython""")
def _extra_static_paths_default(self):
return [os.path.join(self.profile_dir.location, 'static')]
@property
def static_file_path(self):
"""return extra paths + the default location"""
return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""")
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_url_prefix = self.webapp_settings.get("static_url_prefix",
"/static/")
try:
mathjax = filefind(os.path.join('mathjax', 'MathJax.js'),
self.static_file_path)
except IOError:
if self.certfile:
# HTTPS: load from Rackspace CDN, because SSL certificate requires it
base = u"https://c328740.ssl.cf1.rackcdn.com"
else:
base = u"http://cdn.mathjax.org"
url = base + u"/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
else:
self.log.info("Using local MathJax from %s" % mathjax)
return static_url_prefix + u"mathjax/MathJax.js"
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
notebook_manager_class = DottedObjectName(
'IPython.frontend.html.notebook.filenbmanager.FileNotebookManager',
config=True,
help='The notebook manager class to use.')
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if argv is None:
argv = sys.argv[1:]
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" %
self.name)
if self.extra_args:
f = os.path.abspath(self.extra_args[0])
if os.path.isdir(f):
nbdir = f
else:
self.file_to_run = f
nbdir = os.path.dirname(f)
self.config.NotebookManager.notebook_dir = nbdir
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_manager = MappingKernelManager(
config=self.config,
log=self.log,
kernel_argv=self.kernel_argv,
connection_dir=self.profile_dir.security_dir,
)
kls = import_item(self.notebook_manager_class)
self.notebook_manager = kls(config=self.config, log=self.log)
self.notebook_manager.log_info()
self.notebook_manager.load_notebook_names()
self.cluster_manager = ClusterManager(config=self.config, log=self.log)
self.cluster_manager.update_profiles()
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(self, self.kernel_manager,
self.notebook_manager,
self.cluster_manager, self.log,
self.base_project_url,
self.webapp_settings)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app,
ssl_options=ssl_options)
if not self.ip:
warning = "WARNING: The notebook server is listening on all IP addresses"
if ssl_options is None:
self.log.critical(warning + " and not using encryption. This"
"is not recommended.")
if not self.password and not self.read_only:
self.log.critical(
warning + "and not using authentication."
"This is highly insecure and not recommended.")
success = None
for port in random_ports(self.port, self.port_retries + 1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
if e.errno != errno.EADDRINUSE:
raise
self.log.info(
'The port %i is already in use, trying another random port.'
% port)
else:
self.port = port
success = True
break
if not success:
self.log.critical(
'ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
def init_signal(self):
# FIXME: remove this check when pyzmq dependency is >= 2.1.11
# safely extract zmq version info:
try:
zmq_v = zmq.pyzmq_version_info()
except AttributeError:
zmq_v = [int(n) for n in re.findall(r'\d+', zmq.__version__)]
if 'dev' in zmq.__version__:
zmq_v.append(999)
zmq_v = tuple(zmq_v)
if zmq_v >= (2, 1, 9) and not sys.platform.startswith('win'):
# This won't work with 2.1.7 and
# 2.1.9-10 will log ugly 'Interrupted system call' messages,
# but it will work
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
# FIXME: remove this delay when pyzmq dependency is >= 2.1.11
time.sleep(0.1)
sys.stdout.write("Shutdown Notebook Server (y/[n])? ")
sys.stdout.flush()
r, w, x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y'):
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.instance().stop()
return
else:
print "No answer for 5s:",
print "resuming operation..."
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.instance().stop()
@catch_config_error
def initialize(self, argv=None):
self.init_logging()
super(NotebookApp, self).initialize(argv)
self.init_configurables()
self.init_webapp()
self.init_signal()
def cleanup_kernels(self):
"""Shutdown all kernels.
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
self.kernel_manager.shutdown_all()
def start(self):
ip = self.ip if self.ip else '[all ip addresses on your system]'
proto = 'https' if self.certfile else 'http'
info = self.log.info
info("The IPython %s Notebook is running at: %s://%s:%i%s" %
(__version__, proto, ip, self.port, self.base_project_url))
info("Use Control-C to stop this server and shut down all kernels.")
if self.open_browser or self.file_to_run:
ip = self.ip or LOCALHOST
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
name, _ = os.path.splitext(os.path.basename(self.file_to_run))
url = self.notebook_manager.rev_mapping.get(name, '')
else:
url = ''
if browser:
b = lambda: browser.open("%s://%s:%i%s%s" % (
proto, ip, self.port, self.base_project_url, url),
new=2)
threading.Thread(target=b).start()
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
class WinHPCJob(Configurable):
job_id = Unicode('')
job_name = Unicode('MyJob', config=True)
min_cores = Int(1, config=True)
max_cores = Int(1, config=True)
min_sockets = Int(1, config=True)
max_sockets = Int(1, config=True)
min_nodes = Int(1, config=True)
max_nodes = Int(1, config=True)
unit_type = Unicode("Core", config=True)
auto_calculate_min = Bool(True, config=True)
auto_calculate_max = Bool(True, config=True)
run_until_canceled = Bool(False, config=True)
is_exclusive = Bool(False, config=True)
username = Unicode(find_username(), config=True)
job_type = Unicode('Batch', config=True)
priority = Enum(
('Lowest', 'BelowNormal', 'Normal', 'AboveNormal', 'Highest'),
default_value='Highest',
config=True)
requested_nodes = Unicode('', config=True)
project = Unicode('IPython', config=True)
xmlns = Unicode('http://schemas.microsoft.com/HPCS2008/scheduler/')
version = Unicode("2.000")
tasks = List([])
@property
def owner(self):
return self.username
def _write_attr(self, root, attr, key):
s = as_str(getattr(self, attr, ''))
if s:
root.set(key, s)
def as_element(self):
# We have to add _A_ type things to get the right order than
# the MSFT XML parser expects.
root = ET.Element('Job')
self._write_attr(root, 'version', '_A_Version')
self._write_attr(root, 'job_name', '_B_Name')
self._write_attr(root, 'unit_type', '_C_UnitType')
self._write_attr(root, 'min_cores', '_D_MinCores')
self._write_attr(root, 'max_cores', '_E_MaxCores')
self._write_attr(root, 'min_sockets', '_F_MinSockets')
self._write_attr(root, 'max_sockets', '_G_MaxSockets')
self._write_attr(root, 'min_nodes', '_H_MinNodes')
self._write_attr(root, 'max_nodes', '_I_MaxNodes')
self._write_attr(root, 'run_until_canceled', '_J_RunUntilCanceled')
self._write_attr(root, 'is_exclusive', '_K_IsExclusive')
self._write_attr(root, 'username', '_L_UserName')
self._write_attr(root, 'job_type', '_M_JobType')
self._write_attr(root, 'priority', '_N_Priority')
self._write_attr(root, 'requested_nodes', '_O_RequestedNodes')
self._write_attr(root, 'auto_calculate_max', '_P_AutoCalculateMax')
self._write_attr(root, 'auto_calculate_min', '_Q_AutoCalculateMin')
self._write_attr(root, 'project', '_R_Project')
self._write_attr(root, 'owner', '_S_Owner')
self._write_attr(root, 'xmlns', '_T_xmlns')
dependencies = ET.SubElement(root, "Dependencies")
etasks = ET.SubElement(root, "Tasks")
for t in self.tasks:
etasks.append(t.as_element())
return root
def tostring(self):
"""Return the string representation of the job description XML."""
root = self.as_element()
indent(root)
txt = ET.tostring(root, encoding="utf-8")
# Now remove the tokens used to order the attributes.
txt = re.sub(r'_[A-Z]_', '', txt)
txt = '<?xml version="1.0" encoding="utf-8"?>\n' + txt
return txt
def write(self, filename):
"""Write the XML job description to a file."""
txt = self.tostring()
with open(filename, 'w') as f:
f.write(txt)
def add_task(self, task):
"""Add a task to the job.
Parameters
----------
task : :class:`WinHPCTask`
The task object to add.
"""
self.tasks.append(task)
class InteractiveShellEmbed(TerminalInteractiveShell):
dummy_mode = Bool(False)
exit_msg = Unicode('')
embedded = CBool(True)
embedded_active = CBool(True)
# Like the base class display_banner is not configurable, but here it
# is True by default.
display_banner = CBool(True)
exit_msg = Unicode()
def __init__(self, **kw):
if kw.get('user_global_ns', None) is not None:
warnings.warn(
"user_global_ns has been replaced by user_module. The\
parameter will be ignored.", DeprecationWarning)
super(InteractiveShellEmbed, self).__init__(**kw)
# don't use the ipython crash handler so that user exceptions aren't
# trapped
sys.excepthook = ultratb.FormattedTB(color_scheme=self.colors,
mode=self.xmode,
call_pdb=self.pdb)
def init_sys_modules(self):
pass
def init_magics(self):
super(InteractiveShellEmbed, self).init_magics()
self.register_magics(EmbeddedMagics)
def __call__(self,
header='',
local_ns=None,
module=None,
dummy=None,
stack_depth=1,
global_ns=None,
compile_flags=None):
"""Activate the interactive interpreter.
__call__(self,header='',local_ns=None,module=None,dummy=None) -> Start
the interpreter shell with the given local and global namespaces, and
optionally print a header string at startup.
The shell can be globally activated/deactivated using the
dummy_mode attribute. This allows you to turn off a shell used
for debugging globally.
However, *each* time you call the shell you can override the current
state of dummy_mode with the optional keyword parameter 'dummy'. For
example, if you set dummy mode on with IPShell.dummy_mode = True, you
can still have a specific call work by making it as IPShell(dummy=False).
"""
# If the user has turned it off, go away
if not self.embedded_active:
return
# Normal exits from interactive mode set this flag, so the shell can't
# re-enter (it checks this variable at the start of interactive mode).
self.exit_now = False
# Allow the dummy parameter to override the global __dummy_mode
if dummy or (dummy != 0 and self.dummy_mode):
return
if self.has_readline:
self.set_readline_completer()
# self.banner is auto computed
if header:
self.old_banner2 = self.banner2
self.banner2 = self.banner2 + '\n' + header + '\n'
else:
self.old_banner2 = ''
# Call the embedding code with a stack depth of 1 so it can skip over
# our call and get the original caller's namespaces.
self.mainloop(local_ns,
module,
stack_depth=stack_depth,
global_ns=global_ns,
compile_flags=compile_flags)
self.banner2 = self.old_banner2
if self.exit_msg is not None:
print(self.exit_msg)
def mainloop(self,
local_ns=None,
module=None,
stack_depth=0,
display_banner=None,
global_ns=None,
compile_flags=None):
"""Embeds IPython into a running python program.
Parameters
----------
local_ns, module
Working local namespace (a dict) and module (a module or similar
object). If given as None, they are automatically taken from the scope
where the shell was called, so that program variables become visible.
stack_depth : int
How many levels in the stack to go to looking for namespaces (when
local_ns or module is None). This allows an intermediate caller to
make sure that this function gets the namespace from the intended
level in the stack. By default (0) it will get its locals and globals
from the immediate caller.
compile_flags
A bit field identifying the __future__ features
that are enabled, as passed to the builtin :func:`compile` function.
If given as None, they are automatically taken from the scope where
the shell was called.
"""
if (global_ns is not None) and (module is None):
warnings.warn("global_ns is deprecated, use module instead.",
DeprecationWarning)
module = DummyMod()
module.__dict__ = global_ns
# Get locals and globals from caller
if ((local_ns is None or module is None or compile_flags is None)
and self.default_user_namespaces):
call_frame = sys._getframe(stack_depth).f_back
if local_ns is None:
local_ns = call_frame.f_locals
if module is None:
global_ns = call_frame.f_globals
module = sys.modules[global_ns['__name__']]
if compile_flags is None:
compile_flags = (call_frame.f_code.co_flags
& compilerop.PyCF_MASK)
# Save original namespace and module so we can restore them after
# embedding; otherwise the shell doesn't shut down correctly.
orig_user_module = self.user_module
orig_user_ns = self.user_ns
orig_compile_flags = self.compile.flags
# Update namespaces and fire up interpreter
# The global one is easy, we can just throw it in
if module is not None:
self.user_module = module
# But the user/local one is tricky: ipython needs it to store internal
# data, but we also need the locals. We'll throw our hidden variables
# like _ih and get_ipython() into the local namespace, but delete them
# later.
if local_ns is not None:
reentrant_local_ns = {
k: v
for (k, v) in local_ns.items()
if k not in self.user_ns_hidden.keys()
}
self.user_ns = reentrant_local_ns
self.init_user_ns()
# Compiler flags
if compile_flags is not None:
self.compile.flags = compile_flags
# make sure the tab-completer has the correct frame information, so it
# actually completes using the frame's locals/globals
self.set_completer_frame()
with self.builtin_trap, self.display_trap:
self.interact(display_banner=display_banner)
# now, purge out the local namespace of IPython's hidden variables.
if local_ns is not None:
local_ns.update({
k: v
for (k, v) in self.user_ns.items()
if k not in self.user_ns_hidden.keys()
})
# Restore original namespace so shell can shut down when we exit.
self.user_module = orig_user_module
self.user_ns = orig_user_ns
self.compile.flags = orig_compile_flags
