class MappingKernelManager(MultiKernelManager):
"""A KernelManager that handles notebook mapping and HTTP error handling"""
def _kernel_manager_class_default(self):
return "IPython.kernel.ioloop.IOLoopKernelManager"
kernel_argv = List(Unicode)
#-------------------------------------------------------------------------
# Methods for managing kernels and sessions
#-------------------------------------------------------------------------
def _handle_kernel_died(self, kernel_id):
"""notice that a kernel died"""
self.log.warn("Kernel %s died, removing from map.", kernel_id)
self.remove_kernel(kernel_id)
def start_kernel(self, kernel_id=None, **kwargs):
"""Start a kernel for a session an return its kernel_id.
Parameters
----------
kernel_id : uuid
The uuid to associate the new kernel with. If this
is not None, this kernel will be persistent whenever it is
requested.
"""
if kernel_id is None:
kwargs['extra_arguments'] = self.kernel_argv
kernel_id = super(MappingKernelManager,
self).start_kernel(**kwargs)
self.log.info("Kernel started: %s" % kernel_id)
self.log.debug("Kernel args: %r" % kwargs)
# register callback for failed auto-restart
self.add_restart_callback(
kernel_id,
lambda: self._handle_kernel_died(kernel_id),
'dead',
)
else:
self._check_kernel_id(kernel_id)
self.log.info("Using existing kernel: %s" % kernel_id)
return kernel_id
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now)
def kernel_model(self, kernel_id, ws_url):
"""Return a dictionary of kernel information described in the
JSON standard model."""
self._check_kernel_id(kernel_id)
model = {"id": kernel_id, "ws_url": ws_url}
return model
def list_kernels(self, ws_url):
"""Returns a list of kernel_id's of kernels running."""
kernels = []
kernel_ids = super(MappingKernelManager, self).list_kernel_ids()
for kernel_id in kernel_ids:
model = self.kernel_model(kernel_id, ws_url)
kernels.append(model)
return kernels
# override _check_kernel_id to raise 404 instead of KeyError
def _check_kernel_id(self, kernel_id):
"""Check a that a kernel_id exists and raise 404 if not."""
if kernel_id not in self:
raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
class InteractiveShellApp(Configurable):
"""A Mixin for applications that start InteractiveShell instances.
Provides configurables for loading extensions and executing files
as part of configuring a Shell environment.
The following methods should be called by the :meth:`initialize` method
of the subclass:
- :meth:`init_path`
- :meth:`init_shell` (to be implemented by the subclass)
- :meth:`init_gui_pylab`
- :meth:`init_extensions`
- :meth:`init_code`
"""
extensions = List(
Unicode,
config=True,
help="A list of dotted module names of IPython extensions to load.")
extra_extension = Unicode(
'',
config=True,
help="dotted module name of an IPython extension to load.")
def _extra_extension_changed(self, name, old, new):
if new:
# add to self.extensions
self.extensions.append(new)
# Extensions that are always loaded (not configurable)
default_extensions = List(Unicode, [u'storemagic'], config=False)
exec_files = List(Unicode,
config=True,
help="""List of files to run at IPython startup.""")
file_to_run = Unicode('', config=True, help="""A file to be run""")
exec_lines = List(Unicode,
config=True,
help="""lines of code to run at IPython startup.""")
code_to_run = Unicode('',
config=True,
help="Execute the given command string.")
module_to_run = Unicode('',
config=True,
help="Run the module as a script.")
gui = CaselessStrEnum(
('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx'),
config=True,
help=
"Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx')."
)
pylab = CaselessStrEnum(
['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
config=True,
help="""Pre-load matplotlib and numpy for interactive use,
selecting a particular matplotlib backend and loop integration.
""")
pylab_import_all = Bool(
True,
config=True,
help="""If true, an 'import *' is done from numpy and pylab,
when using pylab""")
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
def init_path(self):
"""Add current working directory, '', to sys.path"""
if sys.path[0] != '':
sys.path.insert(0, '')
def init_shell(self):
raise NotImplementedError("Override in subclasses")
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
if self.gui or self.pylab:
shell = self.shell
try:
if self.pylab:
gui, backend = pylabtools.find_gui_and_backend(self.pylab)
self.log.info("Enabling GUI event loop integration, "
"toolkit=%s, pylab=%s" % (gui, self.pylab))
shell.enable_pylab(gui,
import_all=self.pylab_import_all,
welcome_message=True)
else:
self.log.info("Enabling GUI event loop integration, "
"toolkit=%s" % self.gui)
shell.enable_gui(self.gui)
except Exception:
self.log.warn("GUI event loop or pylab initialization failed")
self.shell.showtraceback()
def init_extensions(self):
"""Load all IPython extensions in IPythonApp.extensions.
This uses the :meth:`ExtensionManager.load_extensions` to load all
the extensions listed in ``self.extensions``.
"""
try:
self.log.debug("Loading IPython extensions...")
extensions = self.default_extensions + self.extensions
for ext in extensions:
try:
self.log.info("Loading IPython extension: %s" % ext)
self.shell.extension_manager.load_extension(ext)
except:
self.log.warn("Error in loading extension: %s" % ext +
"\nCheck your config files in %s" %
self.profile_dir.location)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in loading extensions:")
self.shell.showtraceback()
def init_code(self):
"""run the pre-flight code, specified via exec_lines"""
self._run_startup_files()
self._run_exec_lines()
self._run_exec_files()
self._run_cmd_line_code()
self._run_module()
# flush output, so itwon't be attached to the first cell
sys.stdout.flush()
sys.stderr.flush()
# Hide variables defined here from %who etc.
self.shell.user_ns_hidden.update(self.shell.user_ns)
def _run_exec_lines(self):
"""Run lines of code in IPythonApp.exec_lines in the user's namespace."""
if not self.exec_lines:
return
try:
self.log.debug("Running code from IPythonApp.exec_lines...")
for line in self.exec_lines:
try:
self.log.info("Running code in user namespace: %s" % line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user "
"namespace: %s" % line)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
self.shell.showtraceback()
def _exec_file(self, fname):
try:
full_filename = filefind(fname, [u'.', self.ipython_dir])
except IOError as e:
self.log.warn("File not found: %r" % fname)
return
# Make sure that the running script gets a proper sys.argv as if it
# were run from a system shell.
save_argv = sys.argv
sys.argv = [full_filename] + self.extra_args[1:]
# protect sys.argv from potential unicode strings on Python 2:
if not py3compat.PY3:
sys.argv = [py3compat.cast_bytes(a) for a in sys.argv]
try:
if os.path.isfile(full_filename):
self.log.info("Running file in user namespace: %s" %
full_filename)
if full_filename.endswith('.ipy'):
self.shell.safe_execfile_ipy(full_filename)
else:
# default to python, even without extension
self.shell.safe_execfile(full_filename, self.shell.user_ns)
finally:
sys.argv = save_argv
def _run_startup_files(self):
"""Run files from profile startup directory"""
startup_dir = self.profile_dir.startup_dir
startup_files = glob.glob(os.path.join(startup_dir, '*.py'))
startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
if not startup_files:
return
self.log.debug("Running startup files from %s...", startup_dir)
try:
for fname in sorted(startup_files):
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling startup files:")
self.shell.showtraceback()
def _run_exec_files(self):
"""Run files from IPythonApp.exec_files"""
if not self.exec_files:
return
self.log.debug("Running files in IPythonApp.exec_files...")
try:
for fname in self.exec_files:
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling IPythonApp.exec_files:")
self.shell.showtraceback()
def _run_cmd_line_code(self):
"""Run code or file specified at the command-line"""
if self.code_to_run:
line = self.code_to_run
try:
self.log.info("Running code given at command line (c=): %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user namespace: %s" %
line)
self.shell.showtraceback()
# Like Python itself, ignore the second if the first of these is present
elif self.file_to_run:
fname = self.file_to_run
try:
self._exec_file(fname)
except:
self.log.warn("Error in executing file in user namespace: %s" %
fname)
self.shell.showtraceback()
def _run_module(self):
"""Run module specified at the command-line."""
if self.module_to_run:
# Make sure that the module gets a proper sys.argv as if it were
# run using `python -m`.
save_argv = sys.argv
sys.argv = [sys.executable] + self.extra_args
try:
self.shell.safe_run_module(self.module_to_run,
self.shell.user_ns)
finally:
sys.argv = save_argv
class MappingKernelManager(MultiKernelManager):
"""A KernelManager that handles notebok mapping and HTTP error handling"""
kernel_argv = List(Unicode)
time_to_dead = Float(3.0, config=True, help="""Kernel heartbeat interval in seconds.""")
first_beat = Float(5.0, config=True, help="Delay (in seconds) before sending first heartbeat.")
max_msg_size = Integer(65536, config=True, help="""
The max raw message size accepted from the browser
over a WebSocket connection.
""")
_notebook_mapping = Dict()
#-------------------------------------------------------------------------
# Methods for managing kernels and sessions
#-------------------------------------------------------------------------
def kernel_for_notebook(self, notebook_id):
"""Return the kernel_id for a notebook_id or None."""
return self._notebook_mapping.get(notebook_id)
def set_kernel_for_notebook(self, notebook_id, kernel_id):
"""Associate a notebook with a kernel."""
if notebook_id is not None:
self._notebook_mapping[notebook_id] = kernel_id
def notebook_for_kernel(self, kernel_id):
"""Return the notebook_id for a kernel_id or None."""
notebook_ids = [k for k, v in self._notebook_mapping.iteritems() if v == kernel_id]
if len(notebook_ids) == 1:
return notebook_ids[0]
else:
return None
def delete_mapping_for_kernel(self, kernel_id):
"""Remove the kernel/notebook mapping for kernel_id."""
notebook_id = self.notebook_for_kernel(kernel_id)
if notebook_id is not None:
del self._notebook_mapping[notebook_id]
def start_kernel(self, notebook_id=None):
"""Start a kernel for a notebok an return its kernel_id.
Parameters
----------
notebook_id : uuid
The uuid of the notebook to associate the new kernel with. If this
is not None, this kernel will be persistent whenever the notebook
requests a kernel.
"""
kernel_id = self.kernel_for_notebook(notebook_id)
if kernel_id is None:
kwargs = dict()
kwargs['extra_arguments'] = self.kernel_argv
kernel_id = super(MappingKernelManager, self).start_kernel(**kwargs)
self.set_kernel_for_notebook(notebook_id, kernel_id)
self.log.info("Kernel started: %s" % kernel_id)
self.log.debug("Kernel args: %r" % kwargs)
else:
self.log.info("Using existing kernel: %s" % kernel_id)
return kernel_id
def kill_kernel(self, kernel_id):
"""Kill a kernel and remove its notebook association."""
self._check_kernel_id(kernel_id)
super(MappingKernelManager, self).kill_kernel(kernel_id)
self.delete_mapping_for_kernel(kernel_id)
self.log.info("Kernel killed: %s" % kernel_id)
def interrupt_kernel(self, kernel_id):
"""Interrupt a kernel."""
self._check_kernel_id(kernel_id)
super(MappingKernelManager, self).interrupt_kernel(kernel_id)
self.log.info("Kernel interrupted: %s" % kernel_id)
def restart_kernel(self, kernel_id):
"""Restart a kernel while keeping clients connected."""
self._check_kernel_id(kernel_id)
km = self.get_kernel(kernel_id)
km.restart_kernel(now=True)
self.log.info("Kernel restarted: %s" % kernel_id)
return kernel_id
# the following remains, in case the KM restart machinery is
# somehow unacceptable
# Get the notebook_id to preserve the kernel/notebook association.
notebook_id = self.notebook_for_kernel(kernel_id)
# Create the new kernel first so we can move the clients over.
new_kernel_id = self.start_kernel()
# Now kill the old kernel.
self.kill_kernel(kernel_id)
# Now save the new kernel/notebook association. We have to save it
# after the old kernel is killed as that will delete the mapping.
self.set_kernel_for_notebook(notebook_id, new_kernel_id)
self.log.info("Kernel restarted: %s" % new_kernel_id)
return new_kernel_id
def create_iopub_stream(self, kernel_id):
"""Create a new iopub stream."""
self._check_kernel_id(kernel_id)
return super(MappingKernelManager, self).create_iopub_stream(kernel_id)
def create_shell_stream(self, kernel_id):
"""Create a new shell stream."""
self._check_kernel_id(kernel_id)
return super(MappingKernelManager, self).create_shell_stream(kernel_id)
def create_hb_stream(self, kernel_id):
"""Create a new hb stream."""
self._check_kernel_id(kernel_id)
return super(MappingKernelManager, self).create_hb_stream(kernel_id)
def _check_kernel_id(self, kernel_id):
"""Check a that a kernel_id exists and raise 404 if not."""
if kernel_id not in self:
raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
class SSHLauncher(LocalProcessLauncher):
"""A minimal launcher for ssh.
To be useful this will probably have to be extended to use the ``sshx``
idea for environment variables. There could be other things this needs
as well.
"""
ssh_cmd = List(['ssh'], config=True, help="command for starting ssh")
ssh_args = List(['-tt'], config=True, help="args to pass to ssh")
scp_cmd = List(['scp'], config=True, help="command for sending files")
program = List(['date'], help="Program to launch via ssh")
program_args = List([], help="args to pass to remote program")
hostname = Unicode('',
config=True,
help="hostname on which to launch the program")
user = Unicode('', config=True, help="username for ssh")
location = Unicode('',
config=True,
help="user@hostname location for ssh in one setting")
to_fetch = List(
[],
config=True,
help="List of (remote, local) files to fetch after starting")
to_send = List(
[],
config=True,
help="List of (local, remote) files to send before starting")
def _hostname_changed(self, name, old, new):
if self.user:
self.location = u'%s@%s' % (self.user, new)
else:
self.location = new
def _user_changed(self, name, old, new):
self.location = u'%s@%s' % (new, self.hostname)
def find_args(self):
return self.ssh_cmd + self.ssh_args + [self.location] + \
list(map(pipes.quote, self.program + self.program_args))
def _send_file(self, local, remote):
"""send a single file"""
remote = "%s:%s" % (self.location, remote)
for i in range(10):
if not os.path.exists(local):
self.log.debug("waiting for %s" % local)
time.sleep(1)
else:
break
self.log.info("sending %s to %s", local, remote)
check_output(self.scp_cmd + [local, remote])
def send_files(self):
"""send our files (called before start)"""
if not self.to_send:
return
for local_file, remote_file in self.to_send:
self._send_file(local_file, remote_file)
def _fetch_file(self, remote, local):
"""fetch a single file"""
full_remote = "%s:%s" % (self.location, remote)
self.log.info("fetching %s from %s", local, full_remote)
for i in range(10):
# wait up to 10s for remote file to exist
check = check_output(self.ssh_cmd + self.ssh_args + \
[self.location, 'test -e', remote, "&& echo 'yes' || echo 'no'"])
check = check.strip()
if check == 'no':
time.sleep(1)
elif check == 'yes':
break
check_output(self.scp_cmd + [full_remote, local])
def fetch_files(self):
"""fetch remote files (called after start)"""
if not self.to_fetch:
return
for remote_file, local_file in self.to_fetch:
self._fetch_file(remote_file, local_file)
def start(self, hostname=None, user=None):
if hostname is not None:
self.hostname = hostname
if user is not None:
self.user = user
self.send_files()
super(SSHLauncher, self).start()
self.fetch_files()
def signal(self, sig):
if self.state == 'running':
# send escaped ssh connection-closer
self.process.stdin.write('~.')
self.process.stdin.flush()
class IPEngineApp(BaseParallelApplication):
name = 'ipengine'
description = _description
examples = _examples
config_file_name = Unicode(default_config_file_name)
classes = List([ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI])
startup_script = Unicode(u'', config=True,
help='specify a script to be run at startup')
startup_command = Unicode('', config=True,
help='specify a command to be run at startup')
url_file = Unicode(u'', config=True,
help="""The full location of the file containing the connection information for
the controller. If this is not given, the file must be in the
security directory of the cluster directory. This location is
resolved using the `profile` or `profile_dir` options.""",
)
wait_for_url_file = Float(5, config=True,
help="""The maximum number of seconds to wait for url_file to exist.
This is useful for batch-systems and shared-filesystems where the
controller and engine are started at the same time and it
may take a moment for the controller to write the connector files.""")
url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
def _cluster_id_changed(self, name, old, new):
if new:
base = 'ipcontroller-%s' % new
else:
base = 'ipcontroller'
self.url_file_name = "%s-engine.json" % base
log_url = Unicode('', config=True,
help="""The URL for the iploggerapp instance, for forwarding
logging to a central location.""")
# an IPKernelApp instance, used to setup listening for shell frontends
kernel_app = Instance(IPKernelApp)
aliases = Dict(aliases)
flags = Dict(flags)
@property
def kernel(self):
"""allow access to the Kernel object, so I look like IPKernelApp"""
return self.engine.kernel
def find_url_file(self):
"""Set the url file.
Here we don't try to actually see if it exists for is valid as that
is hadled by the connection logic.
"""
config = self.config
# Find the actual controller key file
if not self.url_file:
self.url_file = os.path.join(
self.profile_dir.security_dir,
self.url_file_name
)
def load_connector_file(self):
"""load config from a JSON connector file,
at a *lower* priority than command-line/config files.
"""
self.log.info("Loading url_file %r", self.url_file)
config = self.config
with open(self.url_file) as f:
d = json.loads(f.read())
# allow hand-override of location for disambiguation
# and ssh-server
try:
config.EngineFactory.location
except AttributeError:
config.EngineFactory.location = d['location']
try:
config.EngineFactory.sshserver
except AttributeError:
config.EngineFactory.sshserver = d.get('ssh')
location = config.EngineFactory.location
proto, ip = d['interface'].split('://')
ip = disambiguate_ip_address(ip, location)
d['interface'] = '%s://%s' % (proto, ip)
# DO NOT allow override of basic URLs, serialization, or exec_key
# JSON file takes top priority there
config.Session.key = cast_bytes(d['exec_key'])
config.EngineFactory.url = d['interface'] + ':%i' % d['registration']
config.Session.packer = d['pack']
config.Session.unpacker = d['unpack']
self.log.debug("Config changed:")
self.log.debug("%r", config)
self.connection_info = d
def bind_kernel(self, **kwargs):
"""Promote engine to listening kernel, accessible to frontends."""
if self.kernel_app is not None:
return
self.log.info("Opening ports for direct connections as an IPython kernel")
kernel = self.kernel
kwargs.setdefault('config', self.config)
kwargs.setdefault('log', self.log)
kwargs.setdefault('profile_dir', self.profile_dir)
kwargs.setdefault('session', self.engine.session)
app = self.kernel_app = IPKernelApp(**kwargs)
# allow IPKernelApp.instance():
IPKernelApp._instance = app
app.init_connection_file()
# relevant contents of init_sockets:
app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port)
app.log.debug("shell ROUTER Channel on port: %i", app.shell_port)
app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port)
app.log.debug("iopub PUB Channel on port: %i", app.iopub_port)
kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER)
app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port)
app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port)
# start the heartbeat, and log connection info:
app.init_heartbeat()
app.log_connection_info()
app.write_connection_file()
def init_engine(self):
# This is the working dir by now.
sys.path.insert(0, '')
config = self.config
# print config
self.find_url_file()
# was the url manually specified?
keys = set(self.config.EngineFactory.keys())
keys = keys.union(set(self.config.RegistrationFactory.keys()))
if keys.intersection(set(['ip', 'url', 'port'])):
# Connection info was specified, don't wait for the file
url_specified = True
self.wait_for_url_file = 0
else:
url_specified = False
if self.wait_for_url_file and not os.path.exists(self.url_file):
self.log.warn("url_file %r not found", self.url_file)
self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file)
tic = time.time()
while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
# wait for url_file to exist, or until time limit
time.sleep(0.1)
if os.path.exists(self.url_file):
self.load_connector_file()
elif not url_specified:
self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
self.exit(1)
try:
exec_lines = config.IPKernelApp.exec_lines
except AttributeError:
try:
exec_lines = config.InteractiveShellApp.exec_lines
except AttributeError:
exec_lines = config.IPKernelApp.exec_lines = []
try:
exec_files = config.IPKernelApp.exec_files
except AttributeError:
try:
exec_files = config.InteractiveShellApp.exec_files
except AttributeError:
exec_files = config.IPKernelApp.exec_files = []
if self.startup_script:
exec_files.append(self.startup_script)
if self.startup_command:
exec_lines.append(self.startup_command)
# Create the underlying shell class and Engine
# shell_class = import_item(self.master_config.Global.shell_class)
# print self.config
try:
self.engine = EngineFactory(config=config, log=self.log,
connection_info=self.connection_info,
)
except:
self.log.error("Couldn't start the Engine", exc_info=True)
self.exit(1)
def forward_logging(self):
if self.log_url:
self.log.info("Forwarding logging to %s", self.log_url)
context = self.engine.context
lsock = context.socket(zmq.PUB)
lsock.connect(self.log_url)
handler = EnginePUBHandler(self.engine, lsock)
handler.setLevel(self.log_level)
self.log.addHandler(handler)
def init_mpi(self):
global mpi
self.mpi = MPI(config=self.config)
mpi_import_statement = self.mpi.init_script
if mpi_import_statement:
try:
self.log.info("Initializing MPI:")
self.log.info(mpi_import_statement)
exec mpi_import_statement in globals()
except:
mpi = None
else:
mpi = None
@catch_config_error
def initialize(self, argv=None):
super(IPEngineApp, self).initialize(argv)
self.init_mpi()
self.init_engine()
self.forward_logging()
def start(self):
self.engine.start()
try:
self.engine.loop.start()
except KeyboardInterrupt:
self.log.critical("Engine Interrupted, shutting down...\n")
class Application(SingletonConfigurable):
"""A singleton application with full configuration support."""
# The name of the application, will usually match the name of the command
# line application
name = Unicode(u'application')
# The description of the application that is printed at the beginning
# of the help.
description = Unicode(u'This is an application.')
# default section descriptions
option_description = Unicode(option_description)
keyvalue_description = Unicode(keyvalue_description)
subcommand_description = Unicode(subcommand_description)
# The usage and example string that goes at the end of the help string.
examples = Unicode()
# A sequence of Configurable subclasses whose config=True attributes will
# be exposed at the command line.
classes = List([])
# The version string of this application.
version = Unicode(u'0.0')
# The log level for the application
log_level = Enum(
(0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'),
default_value=logging.WARN,
config=True,
help="Set the log level by value or name.")
def _log_level_changed(self, name, old, new):
"""Adjust the log level when log_level is set."""
if isinstance(new, basestring):
new = getattr(logging, new)
self.log_level = new
self.log.setLevel(new)
log_format = Unicode(
"[%(name)s] %(message)s",
config=True,
help="The Logging format template",
)
def _log_format_changed(self, name, old, new):
"""Change the log formatter when log_format is set."""
_log_handler = self.log.handlers[0]
_log_formatter = logging.Formatter(new)
_log_handler.setFormatter(_log_formatter)
log = Instance(logging.Logger)
def _log_default(self):
"""Start logging for this application.
The default is to log to stderr using a StreamHandler, if no default
handler already exists. The log level starts at logging.WARN, but this
can be adjusted by setting the ``log_level`` attribute.
"""
log = logging.getLogger(self.__class__.__name__)
log.setLevel(self.log_level)
log.propagate = False
_log = log # copied from Logger.hasHandlers() (new in Python 3.2)
while _log:
if _log.handlers:
return log
if not _log.propagate:
break
else:
_log = _log.parent
if sys.executable.endswith('pythonw.exe'):
# this should really go to a file, but file-logging is only
# hooked up in parallel applications
_log_handler = logging.StreamHandler(open(os.devnull, 'w'))
else:
_log_handler = logging.StreamHandler()
_log_formatter = logging.Formatter(self.log_format)
_log_handler.setFormatter(_log_formatter)
log.addHandler(_log_handler)
return log
# the alias map for configurables
aliases = Dict({'log-level': 'Application.log_level'})
# flags for loading Configurables or store_const style flags
# flags are loaded from this dict by '--key' flags
# this must be a dict of two-tuples, the first element being the Config/dict
# and the second being the help string for the flag
flags = Dict()
def _flags_changed(self, name, old, new):
"""ensure flags dict is valid"""
for key, value in new.iteritems():
assert len(value) == 2, "Bad flag: %r:%s" % (key, value)
assert isinstance(value[0],
(dict, Config)), "Bad flag: %r:%s" % (key, value)
assert isinstance(value[1],
basestring), "Bad flag: %r:%s" % (key, value)
# subcommands for launching other applications
# if this is not empty, this will be a parent Application
# this must be a dict of two-tuples,
# the first element being the application class/import string
# and the second being the help string for the subcommand
subcommands = Dict()
# parse_command_line will initialize a subapp, if requested
subapp = Instance('IPython.config.application.Application',
allow_none=True)
# extra command-line arguments that don't set config values
extra_args = List(Unicode)
def __init__(self, **kwargs):
SingletonConfigurable.__init__(self, **kwargs)
# Ensure my class is in self.classes, so my attributes appear in command line
# options and config files.
if self.__class__ not in self.classes:
self.classes.insert(0, self.__class__)
def _config_changed(self, name, old, new):
SingletonConfigurable._config_changed(self, name, old, new)
self.log.debug('Config changed:')
self.log.debug(repr(new))
@catch_config_error
def initialize(self, argv=None):
"""Do the basic steps to configure me.
Override in subclasses.
"""
self.parse_command_line(argv)
def start(self):
"""Start the app mainloop.
Override in subclasses.
"""
if self.subapp is not None:
return self.subapp.start()
def print_alias_help(self):
"""Print the alias part of the help."""
if not self.aliases:
return
lines = []
classdict = {}
for cls in self.classes:
# include all parents (up to, but excluding Configurable) in available names
for c in cls.mro()[:-3]:
classdict[c.__name__] = c
for alias, longname in self.aliases.iteritems():
classname, traitname = longname.split('.', 1)
cls = classdict[classname]
trait = cls.class_traits(config=True)[traitname]
help = cls.class_get_trait_help(trait).splitlines()
# reformat first line
help[0] = help[0].replace(longname, alias) + ' (%s)' % longname
if len(alias) == 1:
help[0] = help[0].replace('--%s=' % alias, '-%s ' % alias)
lines.extend(help)
# lines.append('')
print os.linesep.join(lines)
def print_flag_help(self):
"""Print the flag part of the help."""
if not self.flags:
return
lines = []
for m, (cfg, help) in self.flags.iteritems():
prefix = '--' if len(m) > 1 else '-'
lines.append(prefix + m)
lines.append(indent(dedent(help.strip())))
# lines.append('')
print os.linesep.join(lines)
def print_options(self):
if not self.flags and not self.aliases:
return
lines = ['Options']
lines.append('-' * len(lines[0]))
lines.append('')
for p in wrap_paragraphs(self.option_description):
lines.append(p)
lines.append('')
print os.linesep.join(lines)
self.print_flag_help()
self.print_alias_help()
print
def print_subcommands(self):
"""Print the subcommand part of the help."""
if not self.subcommands:
return
lines = ["Subcommands"]
lines.append('-' * len(lines[0]))
lines.append('')
for p in wrap_paragraphs(self.subcommand_description):
lines.append(p)
lines.append('')
for subc, (cls, help) in self.subcommands.iteritems():
lines.append(subc)
if help:
lines.append(indent(dedent(help.strip())))
lines.append('')
print os.linesep.join(lines)
def print_help(self, classes=False):
"""Print the help for each Configurable class in self.classes.
If classes=False (the default), only flags and aliases are printed.
"""
self.print_subcommands()
self.print_options()
if classes:
if self.classes:
print "Class parameters"
print "----------------"
print
for p in wrap_paragraphs(self.keyvalue_description):
print p
print
for cls in self.classes:
cls.class_print_help()
print
else:
print "To see all available configurables, use `--help-all`"
print
def print_description(self):
"""Print the application description."""
for p in wrap_paragraphs(self.description):
print p
print
def print_examples(self):
"""Print usage and examples.
This usage string goes at the end of the command line help string
and should contain examples of the application's usage.
"""
if self.examples:
print "Examples"
print "--------"
print
print indent(dedent(self.examples.strip()))
print
def print_version(self):
"""Print the version string."""
print self.version
def update_config(self, config):
"""Fire the traits events when the config is updated."""
# Save a copy of the current config.
newconfig = deepcopy(self.config)
# Merge the new config into the current one.
newconfig._merge(config)
# Save the combined config as self.config, which triggers the traits
# events.
self.config = newconfig
@catch_config_error
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand with argv."""
subapp, help = self.subcommands.get(subc)
if isinstance(subapp, basestring):
subapp = import_item(subapp)
# clear existing instances
self.__class__.clear_instance()
# instantiate
self.subapp = subapp.instance()
# and initialize subapp
self.subapp.initialize(argv)
def flatten_flags(self):
"""flatten flags and aliases, so cl-args override as expected.
This prevents issues such as an alias pointing to InteractiveShell,
but a config file setting the same trait in TerminalInteraciveShell
getting inappropriate priority over the command-line arg.
Only aliases with exactly one descendent in the class list
will be promoted.
"""
# build a tree of classes in our list that inherit from a particular
# it will be a dict by parent classname of classes in our list
# that are descendents
mro_tree = defaultdict(list)
for cls in self.classes:
clsname = cls.__name__
for parent in cls.mro()[1:-3]:
# exclude cls itself and Configurable,HasTraits,object
mro_tree[parent.__name__].append(clsname)
# flatten aliases, which have the form:
# { 'alias' : 'Class.trait' }
aliases = {}
for alias, cls_trait in self.aliases.iteritems():
cls, trait = cls_trait.split('.', 1)
children = mro_tree[cls]
if len(children) == 1:
# exactly one descendent, promote alias
cls = children[0]
aliases[alias] = '.'.join([cls, trait])
# flatten flags, which are of the form:
# { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
flags = {}
for key, (flagdict, help) in self.flags.iteritems():
newflag = {}
for cls, subdict in flagdict.iteritems():
children = mro_tree[cls]
# exactly one descendent, promote flag section
if len(children) == 1:
cls = children[0]
newflag[cls] = subdict
flags[key] = (newflag, help)
return flags, aliases
@catch_config_error
def parse_command_line(self, argv=None):
"""Parse the command line arguments."""
argv = sys.argv[1:] if argv is None else argv
if argv and argv[0] == 'help':
# turn `ipython help notebook` into `ipython notebook -h`
argv = argv[1:] + ['-h']
if self.subcommands and len(argv) > 0:
# we have subcommands, and one may have been specified
subc, subargv = argv[0], argv[1:]
if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
# it's a subcommand, and *not* a flag or class parameter
return self.initialize_subcommand(subc, subargv)
# Arguments after a '--' argument are for the script IPython may be
# about to run, not IPython iteslf. For arguments parsed here (help and
# version), we want to only search the arguments up to the first
# occurrence of '--', which we're calling interpreted_argv.
try:
interpreted_argv = argv[:argv.index('--')]
except ValueError:
interpreted_argv = argv
if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')):
self.print_description()
self.print_help('--help-all' in interpreted_argv)
self.print_examples()
self.exit(0)
if '--version' in interpreted_argv or '-V' in interpreted_argv:
self.print_version()
self.exit(0)
# flatten flags&aliases, so cl-args get appropriate priority:
flags, aliases = self.flatten_flags()
loader = KVArgParseConfigLoader(argv=argv,
aliases=aliases,
flags=flags)
config = loader.load_config()
self.update_config(config)
# store unparsed args in extra_args
self.extra_args = loader.extra_args
@catch_config_error
def load_config_file(self, filename, path=None):
"""Load a .py based config file by filename and path."""
loader = PyFileConfigLoader(filename, path=path)
try:
config = loader.load_config()
except ConfigFileNotFound:
# problem finding the file, raise
raise
except Exception:
# try to get the full filename, but it will be empty in the
# unlikely event that the error raised before filefind finished
filename = loader.full_filename or filename
# problem while running the file
self.log.error("Exception while loading config file %s",
filename,
exc_info=True)
else:
self.log.debug("Loaded config file: %s", loader.full_filename)
self.update_config(config)
def generate_config_file(self):
"""generate default config file from Configurables"""
lines = ["# Configuration file for %s." % self.name]
lines.append('')
lines.append('c = get_config()')
lines.append('')
for cls in self.classes:
lines.append(cls.class_config_section())
return '\n'.join(lines)
def exit(self, exit_status=0):
self.log.debug("Exiting application: %s" % self.name)
sys.exit(exit_status)
class BatchSystemLauncher(BaseLauncher):
"""Launch an external process using a batch system.
This class is designed to work with UNIX batch systems like PBS, LSF,
GridEngine, etc. The overall model is that there are different commands
like qsub, qdel, etc. that handle the starting and stopping of the process.
This class also has the notion of a batch script. The ``batch_template``
attribute can be set to a string that is a template for the batch script.
This template is instantiated using string formatting. Thus the template can
use {n} fot the number of instances. Subclasses can add additional variables
to the template dict.
"""
# Subclasses must fill these in. See PBSEngineSet
submit_command = List(
[''],
config=True,
help="The name of the command line program used to submit jobs.")
delete_command = List(
[''],
config=True,
help="The name of the command line program used to delete jobs.")
job_id_regexp = CRegExp(
'',
config=True,
help=
"""A regular expression used to get the job id from the output of the
submit_command.""")
batch_template = Unicode(
'',
config=True,
help="The string that is the batch script template itself.")
batch_template_file = Unicode(
u'', config=True, help="The file that contains the batch template.")
batch_file_name = Unicode(
u'batch_script',
config=True,
help="The filename of the instantiated batch script.")
queue = Unicode(u'', config=True, help="The PBS Queue.")
def _queue_changed(self, name, old, new):
self.context[name] = new
n = Integer(1)
_n_changed = _queue_changed
# not configurable, override in subclasses
# PBS Job Array regex
job_array_regexp = CRegExp('')
job_array_template = Unicode('')
# PBS Queue regex
queue_regexp = CRegExp('')
queue_template = Unicode('')
# The default batch template, override in subclasses
default_template = Unicode('')
# The full path to the instantiated batch script.
batch_file = Unicode(u'')
# the format dict used with batch_template:
context = Dict()
def _context_default(self):
"""load the default context with the default values for the basic keys
because the _trait_changed methods only load the context if they
are set to something other than the default value.
"""
return dict(n=1, queue=u'', profile_dir=u'', cluster_id=u'')
# the Formatter instance for rendering the templates:
formatter = Instance(EvalFormatter, (), {})
def find_args(self):
return self.submit_command + [self.batch_file]
def __init__(self, work_dir=u'.', config=None, **kwargs):
super(BatchSystemLauncher, self).__init__(work_dir=work_dir,
config=config,
**kwargs)
self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
def parse_job_id(self, output):
"""Take the output of the submit command and return the job id."""
m = self.job_id_regexp.search(output)
if m is not None:
job_id = m.group()
else:
raise LauncherError("Job id couldn't be determined: %s" % output)
self.job_id = job_id
self.log.info('Job submitted with job id: %r', job_id)
return job_id
def write_batch_script(self, n):
"""Instantiate and write the batch script to the work_dir."""
self.n = n
# first priority is batch_template if set
if self.batch_template_file and not self.batch_template:
# second priority is batch_template_file
with open(self.batch_template_file) as f:
self.batch_template = f.read()
if not self.batch_template:
# third (last) priority is default_template
self.batch_template = self.default_template
# add jobarray or queue lines to user-specified template
# note that this is *only* when user did not specify a template.
# print self.job_array_regexp.search(self.batch_template)
if not self.job_array_regexp.search(self.batch_template):
self.log.debug("adding job array settings to batch script")
firstline, rest = self.batch_template.split('\n', 1)
self.batch_template = u'\n'.join(
[firstline, self.job_array_template, rest])
# print self.queue_regexp.search(self.batch_template)
if self.queue and not self.queue_regexp.search(
self.batch_template):
self.log.debug("adding PBS queue settings to batch script")
firstline, rest = self.batch_template.split('\n', 1)
self.batch_template = u'\n'.join(
[firstline, self.queue_template, rest])
script_as_string = self.formatter.format(self.batch_template,
**self.context)
self.log.debug('Writing batch script: %s', self.batch_file)
with open(self.batch_file, 'w') as f:
f.write(script_as_string)
os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
def start(self, n):
"""Start n copies of the process using a batch system."""
self.log.debug("Starting %s: %r", self.__class__.__name__, self.args)
# Here we save profile_dir in the context so they
# can be used in the batch script template as {profile_dir}
self.write_batch_script(n)
output = check_output(self.args, env=os.environ)
job_id = self.parse_job_id(output)
self.notify_start(job_id)
return job_id
def stop(self):
output = check_output(self.delete_command + [self.job_id],
env=os.environ)
self.notify_stop(
dict(job_id=self.job_id,
output=output)) # Pass the output of the kill cmd
return output
class NbConvertApp(BaseIPythonApplication):
"""Application used to convert to and from notebook file type (*.ipynb)"""
name = 'ipython-nbconvert'
aliases = nbconvert_aliases
flags = nbconvert_flags
def _log_level_default(self):
return logging.INFO
def _classes_default(self):
classes = [NbConvertBase, ProfileDir]
for pkg in (exporters, transformers, writers):
for name in dir(pkg):
cls = getattr(pkg, name)
if isinstance(cls, type) and issubclass(cls, Configurable):
classes.append(cls)
return classes
description = Unicode(
"""This application is used to convert notebook files (*.ipynb)
to various other formats.
WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""")
output_base = Unicode('',
config=True,
help='''overwrite base name use for output files.
can only be use when converting one notebook at a time.
''')
examples = Unicode("""
The simplest way to use nbconvert is
> ipython nbconvert mynotebook.ipynb
which will convert mynotebook.ipynb to the default format (probably HTML).
You can specify the export format with `--to`.
Options include {0}
> ipython nbconvert --to latex mynotebook.ipnynb
Both HTML and LaTeX support multiple output templates. LaTeX includes
'basic', 'book', and 'article'. HTML includes 'basic' and 'full'. You
can specify the flavor of the format used.
> ipython nbconvert --to html --template basic mynotebook.ipynb
You can also pipe the output to stdout, rather than a file
> ipython nbconvert mynotebook.ipynb --stdout
A post-processor can be used to compile a PDF
> ipython nbconvert mynotebook.ipynb --to latex --post PDF
You can get (and serve) a Reveal.js-powered slideshow
> ipython nbconvert myslides.ipynb --to slides --post serve
Multiple notebooks can be given at the command line in a couple of
different ways:
> ipython nbconvert notebook*.ipynb
> ipython nbconvert notebook1.ipynb notebook2.ipynb
or you can specify the notebooks list in a config file, containing::
c.NbConvertApp.notebooks = ["my_notebook.ipynb"]
> ipython nbconvert --config mycfg.py
""".format(get_export_names()))
# Writer specific variables
writer = Instance('IPython.nbconvert.writers.base.WriterBase',
help="""Instance of the writer class used to write the
results of the conversion.""")
writer_class = DottedObjectName('FilesWriter',
config=True,
help="""Writer class used to write the
results of the conversion""")
writer_aliases = {
'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter',
'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter',
'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'
}
writer_factory = Type()
def _writer_class_changed(self, name, old, new):
if new.lower() in self.writer_aliases:
new = self.writer_aliases[new.lower()]
self.writer_factory = import_item(new)
# Post-processor specific variables
post_processor = Instance(
'IPython.nbconvert.post_processors.base.PostProcessorBase',
help="""Instance of the PostProcessor class used to write the
results of the conversion.""")
post_processor_class = DottedOrNone(
config=True,
help="""PostProcessor class used to write the
results of the conversion""")
post_processor_aliases = {
'pdf': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor',
'serve': 'IPython.nbconvert.post_processors.serve.ServePostProcessor'
}
post_processor_factory = Type()
def _post_processor_class_changed(self, name, old, new):
if new.lower() in self.post_processor_aliases:
new = self.post_processor_aliases[new.lower()]
if new:
self.post_processor_factory = import_item(new)
# Other configurable variables
export_format = CaselessStrEnum(get_export_names(),
default_value="html",
config=True,
help="""The export format to be used.""")
notebooks = List([],
config=True,
help="""List of notebooks to convert.
Wildcards are supported.
Filenames passed positionally will be added to the list.
""")
@catch_config_error
def initialize(self, argv=None):
super(NbConvertApp, self).initialize(argv)
self.init_syspath()
self.init_notebooks()
self.init_writer()
self.init_post_processor()
def init_syspath(self):
"""
Add the cwd to the sys.path ($PYTHONPATH)
"""
sys.path.insert(0, os.getcwd())
def init_notebooks(self):
"""Construct the list of notebooks.
If notebooks are passed on the command-line,
they override notebooks specified in config files.
Glob each notebook to replace notebook patterns with filenames.
"""
# Specifying notebooks on the command-line overrides (rather than adds)
# the notebook list
if self.extra_args:
patterns = self.extra_args
else:
patterns = self.notebooks
# Use glob to replace all the notebook patterns with filenames.
filenames = []
for pattern in patterns:
# Use glob to find matching filenames. Allow the user to convert
# notebooks without having to type the extension.
globbed_files = glob.glob(pattern)
globbed_files.extend(glob.glob(pattern + '.ipynb'))
if not globbed_files:
self.log.warn("pattern %r matched no files", pattern)
for filename in globbed_files:
if not filename in filenames:
filenames.append(filename)
self.notebooks = filenames
def init_writer(self):
"""
Initialize the writer (which is stateless)
"""
self._writer_class_changed(None, self.writer_class, self.writer_class)
self.writer = self.writer_factory(parent=self)
def init_post_processor(self):
"""
Initialize the post_processor (which is stateless)
"""
self._post_processor_class_changed(None, self.post_processor_class,
self.post_processor_class)
if self.post_processor_factory:
self.post_processor = self.post_processor_factory(parent=self)
def start(self):
"""
Ran after initialization completed
"""
super(NbConvertApp, self).start()
self.convert_notebooks()
def convert_notebooks(self):
"""
Convert the notebooks in the self.notebook traitlet
"""
# Export each notebook
conversion_success = 0
if self.output_base != '' and len(self.notebooks) > 1:
self.log.error(
"""UsageError: --output flag or `NbConvertApp.output_base` config option
cannot be used when converting multiple notebooks.
""")
self.exit(1)
exporter = exporter_map[self.export_format](config=self.config)
for notebook_filename in self.notebooks:
self.log.info("Converting notebook %s to %s", notebook_filename,
self.export_format)
# Get a unique key for the notebook and set it in the resources object.
basename = os.path.basename(notebook_filename)
notebook_name = basename[:basename.rfind('.')]
if self.output_base:
notebook_name = self.output_base
resources = {}
resources['unique_key'] = notebook_name
resources['output_files_dir'] = '%s_files' % notebook_name
self.log.info("Support files will be in %s",
os.path.join(resources['output_files_dir'], ''))
# Try to export
try:
output, resources = exporter.from_filename(notebook_filename,
resources=resources)
except ConversionException as e:
self.log.error("Error while converting '%s'",
notebook_filename,
exc_info=True)
self.exit(1)
else:
write_resultes = self.writer.write(output,
resources,
notebook_name=notebook_name)
#Post-process if post processor has been defined.
if hasattr(self, 'post_processor') and self.post_processor:
self.post_processor(write_resultes)
conversion_success += 1
# If nothing was converted successfully, help the user.
if conversion_success == 0:
self.print_help()
sys.exit(-1)
class InteractiveShellApp(Configurable):
"""A Mixin for applications that start InteractiveShell instances.
Provides configurables for loading extensions and executing files
as part of configuring a Shell environment.
The following methods should be called by the :meth:`initialize` method
of the subclass:
- :meth:`init_path`
- :meth:`init_shell` (to be implemented by the subclass)
- :meth:`init_gui_pylab`
- :meth:`init_extensions`
- :meth:`init_code`
"""
extensions = List(
Unicode,
config=True,
help="A list of dotted module names of IPython extensions to load.")
extra_extension = Unicode(
'',
config=True,
help="dotted module name of an IPython extension to load.")
def _extra_extension_changed(self, name, old, new):
if new:
# add to self.extensions
self.extensions.append(new)
# Extensions that are always loaded (not configurable)
default_extensions = List(Unicode, [u'storemagic'], config=False)
hide_initial_ns = Bool(
True,
config=True,
help=
"""Should variables loaded at startup (by startup files, exec_lines, etc.)
be hidden from tools like %who?""")
exec_files = List(Unicode,
config=True,
help="""List of files to run at IPython startup.""")
exec_PYTHONSTARTUP = Bool(
True,
config=True,
help="""Run the file referenced by the PYTHONSTARTUP environment
variable at IPython startup.""")
file_to_run = Unicode('', config=True, help="""A file to be run""")
exec_lines = List(Unicode,
config=True,
help="""lines of code to run at IPython startup.""")
code_to_run = Unicode('',
config=True,
help="Execute the given command string.")
module_to_run = Unicode('',
config=True,
help="Run the module as a script.")
gui = CaselessStrEnum(
gui_keys,
config=True,
help="Enable GUI event loop integration with any of {0}.".format(
gui_keys))
matplotlib = CaselessStrEnum(
backend_keys,
config=True,
help="""Configure matplotlib for interactive use with
the default matplotlib backend.""")
pylab = CaselessStrEnum(
backend_keys,
config=True,
help="""Pre-load matplotlib and numpy for interactive use,
selecting a particular matplotlib backend and loop integration.
""")
pylab_import_all = Bool(
True,
config=True,
help=
"""If true, IPython will populate the user namespace with numpy, pylab, etc.
and an ``import *`` is done from numpy and pylab, when using pylab mode.
When False, pylab mode should not import any names into the user namespace.
""")
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
user_ns = Instance(dict, args=None, allow_none=True)
def _user_ns_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_ns = new
self.shell.init_user_ns()
def init_path(self):
"""Add current working directory, '', to sys.path"""
if sys.path[0] != '':
sys.path.insert(0, '')
def init_shell(self):
raise NotImplementedError("Override in subclasses")
def init_gui_pylab(self):
"""Enable GUI event loop integration, taking pylab into account."""
enable = False
shell = self.shell
if self.pylab:
enable = lambda key: shell.enable_pylab(
key, import_all=self.pylab_import_all)
key = self.pylab
elif self.matplotlib:
enable = shell.enable_matplotlib
key = self.matplotlib
elif self.gui:
enable = shell.enable_gui
key = self.gui
if not enable:
return
try:
r = enable(key)
except ImportError:
self.log.warn(
"Eventloop or matplotlib integration failed. Is matplotlib installed?"
)
self.shell.showtraceback()
return
except Exception:
self.log.warn("GUI event loop or pylab initialization failed")
self.shell.showtraceback()
return
if isinstance(r, tuple):
gui, backend = r[:2]
self.log.info(
"Enabling GUI event loop integration, "
"eventloop=%s, matplotlib=%s", gui, backend)
if key == "auto":
print("Using matplotlib backend: %s" % backend)
else:
gui = r
self.log.info(
"Enabling GUI event loop integration, "
"eventloop=%s", gui)
def init_extensions(self):
"""Load all IPython extensions in IPythonApp.extensions.
This uses the :meth:`ExtensionManager.load_extensions` to load all
the extensions listed in ``self.extensions``.
"""
try:
self.log.debug("Loading IPython extensions...")
extensions = self.default_extensions + self.extensions
for ext in extensions:
try:
self.log.info("Loading IPython extension: %s" % ext)
self.shell.extension_manager.load_extension(ext)
except:
self.log.warn("Error in loading extension: %s" % ext +
"\nCheck your config files in %s" %
self.profile_dir.location)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in loading extensions:")
self.shell.showtraceback()
def init_code(self):
"""run the pre-flight code, specified via exec_lines"""
self._run_startup_files()
self._run_exec_lines()
self._run_exec_files()
# Hide variables defined here from %who etc.
if self.hide_initial_ns:
self.shell.user_ns_hidden.update(self.shell.user_ns)
# command-line execution (ipython -i script.py, ipython -m module)
# should *not* be excluded from %whos
self._run_cmd_line_code()
self._run_module()
# flush output, so itwon't be attached to the first cell
sys.stdout.flush()
sys.stderr.flush()
def _run_exec_lines(self):
"""Run lines of code in IPythonApp.exec_lines in the user's namespace."""
if not self.exec_lines:
return
try:
self.log.debug("Running code from IPythonApp.exec_lines...")
for line in self.exec_lines:
try:
self.log.info("Running code in user namespace: %s" % line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user "
"namespace: %s" % line)
self.shell.showtraceback()
except:
self.log.warn("Unknown error in handling IPythonApp.exec_lines:")
self.shell.showtraceback()
def _exec_file(self, fname):
try:
full_filename = filefind(fname, [u'.', self.ipython_dir])
except IOError as e:
self.log.warn("File not found: %r" % fname)
return
# Make sure that the running script gets a proper sys.argv as if it
# were run from a system shell.
save_argv = sys.argv
sys.argv = [full_filename] + self.extra_args[1:]
# protect sys.argv from potential unicode strings on Python 2:
if not py3compat.PY3:
sys.argv = [py3compat.cast_bytes(a) for a in sys.argv]
try:
if os.path.isfile(full_filename):
self.log.info("Running file in user namespace: %s" %
full_filename)
# Ensure that __file__ is always defined to match Python
# behavior.
with preserve_keys(self.shell.user_ns, '__file__'):
self.shell.user_ns['__file__'] = fname
if full_filename.endswith('.ipy'):
self.shell.safe_execfile_ipy(full_filename)
else:
# default to python, even without extension
self.shell.safe_execfile(full_filename,
self.shell.user_ns)
finally:
sys.argv = save_argv
def _run_startup_files(self):
"""Run files from profile startup directory"""
startup_dir = self.profile_dir.startup_dir
startup_files = []
if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \
not (self.file_to_run or self.code_to_run or self.module_to_run):
python_startup = os.environ['PYTHONSTARTUP']
self.log.debug("Running PYTHONSTARTUP file %s...", python_startup)
try:
self._exec_file(python_startup)
except:
self.log.warn(
"Unknown error in handling PYTHONSTARTUP file %s:",
python_startup)
self.shell.showtraceback()
finally:
# Many PYTHONSTARTUP files set up the readline completions,
# but this is often at odds with IPython's own completions.
# Do not allow PYTHONSTARTUP to set up readline.
if self.shell.has_readline:
self.shell.set_readline_completer()
startup_files += glob.glob(os.path.join(startup_dir, '*.py'))
startup_files += glob.glob(os.path.join(startup_dir, '*.ipy'))
if not startup_files:
return
self.log.debug("Running startup files from %s...", startup_dir)
try:
for fname in sorted(startup_files):
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling startup files:")
self.shell.showtraceback()
def _run_exec_files(self):
"""Run files from IPythonApp.exec_files"""
if not self.exec_files:
return
self.log.debug("Running files in IPythonApp.exec_files...")
try:
for fname in self.exec_files:
self._exec_file(fname)
except:
self.log.warn("Unknown error in handling IPythonApp.exec_files:")
self.shell.showtraceback()
def _run_cmd_line_code(self):
"""Run code or file specified at the command-line"""
if self.code_to_run:
line = self.code_to_run
try:
self.log.info("Running code given at command line (c=): %s" %
line)
self.shell.run_cell(line, store_history=False)
except:
self.log.warn("Error in executing line in user namespace: %s" %
line)
self.shell.showtraceback()
# Like Python itself, ignore the second if the first of these is present
elif self.file_to_run:
fname = self.file_to_run
try:
self._exec_file(fname)
except:
self.log.warn("Error in executing file in user namespace: %s" %
fname)
self.shell.showtraceback()
def _run_module(self):
"""Run module specified at the command-line."""
if self.module_to_run:
# Make sure that the module gets a proper sys.argv as if it were
# run using `python -m`.
save_argv = sys.argv
sys.argv = [sys.executable] + self.extra_args
try:
self.shell.safe_run_module(self.module_to_run,
self.shell.user_ns)
finally:
sys.argv = save_argv
class DOMWidget(Widget):
visible = Bool(True, help="Whether the widget is visible.", sync=True)
_css = List(
sync=True) # Internal CSS property list: (selector, key, value)
def get_css(self, key, selector=""):
"""Get a CSS property of the widget.
Note: This function does not actually request the CSS from the
front-end; Only properties that have been set with set_css can be read.
Parameters
----------
key: unicode
CSS key
selector: unicode (optional)
JQuery selector used when the CSS key/value was set.
"""
if selector in self._css and key in self._css[selector]:
return self._css[selector][key]
else:
return None
def set_css(self, dict_or_key, value=None, selector=''):
"""Set one or more CSS properties of the widget.
This function has two signatures:
- set_css(css_dict, selector='')
- set_css(key, value, selector='')
Parameters
----------
css_dict : dict
CSS key/value pairs to apply
key: unicode
CSS key
value:
CSS value
selector: unicode (optional, kwarg only)
JQuery selector to use to apply the CSS key/value. If no selector
is provided, an empty selector is used. An empty selector makes the
front-end try to apply the css to a default element. The default
element is an attribute unique to each view, which is a DOM element
of the view that should be styled with common CSS (see
`$el_to_style` in the Javascript code).
"""
if value is None:
css_dict = dict_or_key
else:
css_dict = {dict_or_key: value}
for (key, value) in css_dict.items():
# First remove the selector/key pair from the css list if it exists.
# Then add the selector/key pair and new value to the bottom of the
# list.
self._css = [
x for x in self._css if not (x[0] == selector and x[1] == key)
]
self._css += [(selector, key, value)]
self.send_state('_css')
def add_class(self, class_names, selector=""):
"""Add class[es] to a DOM element.
Parameters
----------
class_names: unicode or list
Class name(s) to add to the DOM element(s).
selector: unicode (optional)
JQuery selector to select the DOM element(s) that the class(es) will
be added to.
"""
class_list = class_names
if isinstance(class_list, (list, tuple)):
class_list = ' '.join(class_list)
self.send({
"msg_type": "add_class",
"class_list": class_list,
"selector": selector
})
def remove_class(self, class_names, selector=""):
"""Remove class[es] from a DOM element.
Parameters
----------
class_names: unicode or list
Class name(s) to remove from the DOM element(s).
selector: unicode (optional)
JQuery selector to select the DOM element(s) that the class(es) will
be removed from.
"""
class_list = class_names
if isinstance(class_list, (list, tuple)):
class_list = ' '.join(class_list)
self.send({
"msg_type": "remove_class",
"class_list": class_list,
"selector": selector,
})
class Widget(LoggingConfigurable):
#-------------------------------------------------------------------------
# Class attributes
#-------------------------------------------------------------------------
_widget_construction_callback = None
widgets = {}
@staticmethod
def on_widget_constructed(callback):
"""Registers a callback to be called when a widget is constructed.
The callback must have the following signature:
callback(widget)"""
Widget._widget_construction_callback = callback
@staticmethod
def _call_widget_constructed(widget):
"""Static method, called when a widget is constructed."""
if Widget._widget_construction_callback is not None and callable(
Widget._widget_construction_callback):
Widget._widget_construction_callback(widget)
#-------------------------------------------------------------------------
# Traits
#-------------------------------------------------------------------------
_model_name = Unicode('WidgetModel',
help="""Name of the backbone model
registered in the front-end to create and sync this widget with.""")
_view_name = Unicode(help="""Default view registered in the front-end
to use to represent the widget.""",
sync=True)
_comm = Instance('IPython.kernel.comm.Comm')
msg_throttle = Int(3,
sync=True,
help="""Maximum number of msgs the
front-end can send before receiving an idle msg from the back-end.""")
keys = List()
def _keys_default(self):
return [name for name in self.traits(sync=True)]
_property_lock = Tuple((None, None))
_display_callbacks = Instance(CallbackDispatcher, ())
_msg_callbacks = Instance(CallbackDispatcher, ())
#-------------------------------------------------------------------------
# (Con/de)structor
#-------------------------------------------------------------------------
def __init__(self, **kwargs):
"""Public constructor"""
super(Widget, self).__init__(**kwargs)
self.on_trait_change(self._handle_property_changed, self.keys)
Widget._call_widget_constructed(self)
def __del__(self):
"""Object disposal"""
self.close()
#-------------------------------------------------------------------------
# Properties
#-------------------------------------------------------------------------
@property
def comm(self):
"""Gets the Comm associated with this widget.
If a Comm doesn't exist yet, a Comm will be created automagically."""
if self._comm is None:
# Create a comm.
self._comm = Comm(target_name=self._model_name)
self._comm.on_msg(self._handle_msg)
self._comm.on_close(self._close)
Widget.widgets[self.model_id] = self
# first update
self.send_state()
return self._comm
@property
def model_id(self):
"""Gets the model id of this widget.
If a Comm doesn't exist yet, a Comm will be created automagically."""
return self.comm.comm_id
#-------------------------------------------------------------------------
# Methods
#-------------------------------------------------------------------------
def _close(self):
"""Private close - cleanup objects, registry entries"""
del Widget.widgets[self.model_id]
self._comm = None
def close(self):
"""Close method.
Closes the widget which closes the underlying comm.
When the comm is closed, all of the widget views are automatically
removed from the front-end."""
if self._comm is not None:
self._comm.close()
self._close()
def send_state(self, key=None):
"""Sends the widget state, or a piece of it, to the front-end.
Parameters
----------
key : unicode (optional)
A single property's name to sync with the front-end.
"""
self._send({"method": "update", "state": self.get_state()})
def get_state(self, key=None):
"""Gets the widget state, or a piece of it.
Parameters
----------
key : unicode (optional)
A single property's name to get.
"""
keys = self.keys if key is None else [key]
return {k: self._pack_widgets(getattr(self, k)) for k in keys}
def send(self, content):
"""Sends a custom msg to the widget model in the front-end.
Parameters
----------
content : dict
Content of the message to send.
"""
self._send({"method": "custom", "content": content})
def on_msg(self, callback, remove=False):
"""(Un)Register a custom msg receive callback.
Parameters
----------
callback: callable
callback will be passed two arguments when a message arrives::
callback(widget, content)
remove: bool
True if the callback should be unregistered."""
self._msg_callbacks.register_callback(callback, remove=remove)
def on_displayed(self, callback, remove=False):
"""(Un)Register a widget displayed callback.
Parameters
----------
callback: method handler
Must have a signature of::
callback(widget, **kwargs)
kwargs from display are passed through without modification.
remove: bool
True if the callback should be unregistered."""
self._display_callbacks.register_callback(callback, remove=remove)
#-------------------------------------------------------------------------
# Support methods
#-------------------------------------------------------------------------
@contextmanager
def _lock_property(self, key, value):
"""Lock a property-value pair.
NOTE: This, in addition to the single lock for all state changes, is
flawed. In the future we may want to look into buffering state changes
back to the front-end."""
self._property_lock = (key, value)
try:
yield
finally:
self._property_lock = (None, None)
def _should_send_property(self, key, value):
"""Check the property lock (property_lock)"""
return key != self._property_lock[0] or \
value != self._property_lock[1]
# Event handlers
@_show_traceback
def _handle_msg(self, msg):
"""Called when a msg is received from the front-end"""
data = msg['content']['data']
method = data['method']
if not method in ['backbone', 'custom']:
self.log.error(
'Unknown front-end to back-end widget msg with method "%s"' %
method)
# Handle backbone sync methods CREATE, PATCH, and UPDATE all in one.
if method == 'backbone' and 'sync_data' in data:
sync_data = data['sync_data']
self._handle_receive_state(sync_data) # handles all methods
# Handle a custom msg from the front-end
elif method == 'custom':
if 'content' in data:
self._handle_custom_msg(data['content'])
def _handle_receive_state(self, sync_data):
"""Called when a state is received from the front-end."""
for name in self.keys:
if name in sync_data:
value = self._unpack_widgets(sync_data[name])
with self._lock_property(name, value):
setattr(self, name, value)
def _handle_custom_msg(self, content):
"""Called when a custom msg is received."""
self._msg_callbacks(self, content)
def _handle_property_changed(self, name, old, new):
"""Called when a property has been changed."""
# Make sure this isn't information that the front-end just sent us.
if self._should_send_property(name, new):
# Send new state to front-end
self.send_state(key=name)
def _handle_displayed(self, **kwargs):
"""Called when a view has been displayed for this widget instance"""
self._display_callbacks(self, **kwargs)
def _pack_widgets(self, x):
"""Recursively converts all widget instances to model id strings.
Children widgets will be stored and transmitted to the front-end by
their model ids. Return value must be JSON-able."""
if isinstance(x, dict):
return {k: self._pack_widgets(v) for k, v in x.items()}
elif isinstance(x, (list, tuple)):
return [self._pack_widgets(v) for v in x]
elif isinstance(x, Widget):
return x.model_id
else:
return x # Value must be JSON-able
def _unpack_widgets(self, x):
"""Recursively converts all model id strings to widget instances.
Children widgets will be stored and transmitted to the front-end by
their model ids."""
if isinstance(x, dict):
return {k: self._unpack_widgets(v) for k, v in x.items()}
elif isinstance(x, (list, tuple)):
return [self._unpack_widgets(v) for v in x]
elif isinstance(x, string_types):
return x if x not in Widget.widgets else Widget.widgets[x]
else:
return x
def _ipython_display_(self, **kwargs):
"""Called when `IPython.display.display` is called on the widget."""
# Show view. By sending a display message, the comm is opened and the
# initial state is sent.
self._send({"method": "display"})
self._handle_displayed(**kwargs)
def _send(self, msg):
"""Sends a message to the model in the front-end."""
self.comm.send(msg)
class KernelManager(LoggingConfigurable, ConnectionFileMixin):
"""Manages a single kernel in a subprocess on this host.
This version starts kernels with Popen.
"""
# The PyZMQ Context to use for communication with the kernel.
context = Instance(zmq.Context)
def _context_default(self):
return zmq.Context.instance()
# The Session to use for communication with the kernel.
session = Instance(Session)
def _session_default(self):
return Session(parent=self)
# the class to create with our `client` method
client_class = DottedObjectName(
'IPython.kernel.blocking.BlockingKernelClient')
client_factory = Type()
def _client_class_changed(self, name, old, new):
self.client_factory = import_item(str(new))
# The kernel process with which the KernelManager is communicating.
# generally a Popen instance
kernel = Any()
kernel_cmd = List(Unicode,
config=True,
help="""The Popen Command to launch the kernel.
Override this if you have a custom
""")
def _kernel_cmd_changed(self, name, old, new):
self.ipython_kernel = False
ipython_kernel = Bool(True)
# Protected traits
_launch_args = Any()
_control_socket = Any()
_restarter = Any()
autorestart = Bool(False,
config=True,
help="""Should we autorestart the kernel if it dies.""")
def __del__(self):
self._close_control_socket()
self.cleanup_connection_file()
#--------------------------------------------------------------------------
# Kernel restarter
#--------------------------------------------------------------------------
def start_restarter(self):
pass
def stop_restarter(self):
pass
def add_restart_callback(self, callback, event='restart'):
"""register a callback to be called when a kernel is restarted"""
if self._restarter is None:
return
self._restarter.add_callback(callback, event)
def remove_restart_callback(self, callback, event='restart'):
"""unregister a callback to be called when a kernel is restarted"""
if self._restarter is None:
return
self._restarter.remove_callback(callback, event)
#--------------------------------------------------------------------------
# create a Client connected to our Kernel
#--------------------------------------------------------------------------
def client(self, **kwargs):
"""Create a client configured to connect to our kernel"""
if self.client_factory is None:
self.client_factory = import_item(self.client_class)
kw = {}
kw.update(self.get_connection_info())
kw.update(
dict(
connection_file=self.connection_file,
session=self.session,
parent=self,
))
# add kwargs last, for manual overrides
kw.update(kwargs)
return self.client_factory(**kw)
#--------------------------------------------------------------------------
# Kernel management
#--------------------------------------------------------------------------
def format_kernel_cmd(self, **kw):
"""replace templated args (e.g. {connection_file})"""
if self.kernel_cmd:
cmd = self.kernel_cmd
else:
cmd = make_ipkernel_cmd(
'from IPython.kernel.zmq.kernelapp import main; main()', **kw)
ns = dict(connection_file=self.connection_file)
ns.update(self._launch_args)
pat = re.compile(r'\{([A-Za-z0-9_]+)\}')
def from_ns(match):
"""Get the key out of ns if it's there, otherwise no change."""
return ns.get(match.group(1), match.group())
return [pat.sub(from_ns, arg) for arg in cmd]
def _launch_kernel(self, kernel_cmd, **kw):
"""actually launch the kernel
override in a subclass to launch kernel subprocesses differently
"""
return launch_kernel(kernel_cmd, **kw)
# Control socket used for polite kernel shutdown
def _connect_control_socket(self):
if self._control_socket is None:
self._control_socket = self.connect_control()
self._control_socket.linger = 100
def _close_control_socket(self):
if self._control_socket is None:
return
self._control_socket.close()
self._control_socket = None
def start_kernel(self, **kw):
"""Starts a kernel on this host in a separate process.
If random ports (port=0) are being used, this method must be called
before the channels are created.
Parameters
----------
**kw : optional
keyword arguments that are passed down to build the kernel_cmd
and launching the kernel (e.g. Popen kwargs).
"""
if self.transport == 'tcp' and not is_local_ip(self.ip):
raise RuntimeError(
"Can only launch a kernel on a local interface. "
"Make sure that the '*_address' attributes are "
"configured properly. "
"Currently valid addresses are: %s" % local_ips())
# write connection file / get default ports
self.write_connection_file()
# save kwargs for use in restart
self._launch_args = kw.copy()
# build the Popen cmd
kernel_cmd = self.format_kernel_cmd(**kw)
# launch the kernel subprocess
self.kernel = self._launch_kernel(kernel_cmd,
ipython_kernel=self.ipython_kernel,
**kw)
self.start_restarter()
self._connect_control_socket()
def _send_shutdown_request(self, restart=False):
"""TODO: send a shutdown request via control channel"""
content = dict(restart=restart)
msg = self.session.msg("shutdown_request", content=content)
self.session.send(self._control_socket, msg)
def shutdown_kernel(self, now=False, restart=False):
"""Attempts to the stop the kernel process cleanly.
This attempts to shutdown the kernels cleanly by:
1. Sending it a shutdown message over the shell channel.
2. If that fails, the kernel is shutdown forcibly by sending it
a signal.
Parameters
----------
now : bool
Should the kernel be forcible killed *now*. This skips the
first, nice shutdown attempt.
restart: bool
Will this kernel be restarted after it is shutdown. When this
is True, connection files will not be cleaned up.
"""
# Stop monitoring for restarting while we shutdown.
self.stop_restarter()
# FIXME: Shutdown does not work on Windows due to ZMQ errors!
if sys.platform == 'win32':
self._kill_kernel()
return
if now:
if self.has_kernel:
self._kill_kernel()
else:
# Don't send any additional kernel kill messages immediately, to give
# the kernel a chance to properly execute shutdown actions. Wait for at
# most 1s, checking every 0.1s.
self._send_shutdown_request(restart=restart)
for i in range(10):
if self.is_alive():
time.sleep(0.1)
else:
break
else:
# OK, we've waited long enough.
if self.has_kernel:
self._kill_kernel()
if not restart:
self.cleanup_connection_file()
self.cleanup_ipc_files()
else:
self.cleanup_ipc_files()
def restart_kernel(self, now=False, **kw):
"""Restarts a kernel with the arguments that were used to launch it.
If the old kernel was launched with random ports, the same ports will be
used for the new kernel. The same connection file is used again.
Parameters
----------
now : bool, optional
If True, the kernel is forcefully restarted *immediately*, without
having a chance to do any cleanup action. Otherwise the kernel is
given 1s to clean up before a forceful restart is issued.
In all cases the kernel is restarted, the only difference is whether
it is given a chance to perform a clean shutdown or not.
**kw : optional
Any options specified here will overwrite those used to launch the
kernel.
"""
if self._launch_args is None:
raise RuntimeError("Cannot restart the kernel. "
"No previous call to 'start_kernel'.")
else:
# Stop currently running kernel.
self.shutdown_kernel(now=now, restart=True)
# Start new kernel.
self._launch_args.update(kw)
self.start_kernel(**self._launch_args)
# FIXME: Messages get dropped in Windows due to probable ZMQ bug
# unless there is some delay here.
if sys.platform == 'win32':
time.sleep(0.2)
@property
def has_kernel(self):
"""Has a kernel been started that we are managing."""
return self.kernel is not None
def _kill_kernel(self):
"""Kill the running kernel.
This is a private method, callers should use shutdown_kernel(now=True).
"""
if self.has_kernel:
# Signal the kernel to terminate (sends SIGKILL on Unix and calls
# TerminateProcess() on Win32).
try:
self.kernel.kill()
except OSError as e:
# In Windows, we will get an Access Denied error if the process
# has already terminated. Ignore it.
if sys.platform == 'win32':
if e.winerror != 5:
raise
# On Unix, we may get an ESRCH error if the process has already
# terminated. Ignore it.
else:
from errno import ESRCH
if e.errno != ESRCH:
raise
# Block until the kernel terminates.
self.kernel.wait()
self.kernel = None
else:
raise RuntimeError("Cannot kill kernel. No kernel is running!")
def interrupt_kernel(self):
"""Interrupts the kernel by sending it a signal.
Unlike ``signal_kernel``, this operation is well supported on all
platforms.
"""
if self.has_kernel:
if sys.platform == 'win32':
from .zmq.parentpoller import ParentPollerWindows as Poller
Poller.send_interrupt(self.kernel.win32_interrupt_event)
else:
self.kernel.send_signal(signal.SIGINT)
else:
raise RuntimeError(
"Cannot interrupt kernel. No kernel is running!")
def signal_kernel(self, signum):
"""Sends a signal to the kernel.
Note that since only SIGTERM is supported on Windows, this function is
only useful on Unix systems.
"""
if self.has_kernel:
self.kernel.send_signal(signum)
else:
raise RuntimeError("Cannot signal kernel. No kernel is running!")
def is_alive(self):
"""Is the kernel process still running?"""
if self.has_kernel:
if self.kernel.poll() is None:
return True
else:
return False
else:
# we don't have a kernel
return False
class LenListTrait(HasTraits):
value = List(Int, [0], minlen=1, maxlen=2)
class ListTrait(HasTraits):
value = List(Int)
class Widget(LoggingConfigurable):
#-------------------------------------------------------------------------
# Class attributes
#-------------------------------------------------------------------------
_widget_construction_callback = None
widgets = {}
widget_types = {}
@staticmethod
def on_widget_constructed(callback):
"""Registers a callback to be called when a widget is constructed.
The callback must have the following signature:
callback(widget)"""
Widget._widget_construction_callback = callback
@staticmethod
def _call_widget_constructed(widget):
"""Static method, called when a widget is constructed."""
if Widget._widget_construction_callback is not None and callable(
Widget._widget_construction_callback):
Widget._widget_construction_callback(widget)
@staticmethod
def handle_comm_opened(comm, msg):
"""Static method, called when a widget is constructed."""
widget_class = import_item(msg['content']['data']['widget_class'])
widget = widget_class(comm=comm)
#-------------------------------------------------------------------------
# Traits
#-------------------------------------------------------------------------
_model_module = Unicode(None,
allow_none=True,
help="""A requirejs module name
in which to find _model_name. If empty, look in the global registry."""
)
_model_name = Unicode('WidgetModel',
help="""Name of the backbone model
registered in the front-end to create and sync this widget with.""")
_view_module = Unicode(
help="""A requirejs module in which to find _view_name.
If empty, look in the global registry.""",
sync=True)
_view_name = Unicode(None,
allow_none=True,
help="""Default view registered in the front-end
to use to represent the widget.""",
sync=True)
comm = Instance('IPython.kernel.comm.Comm')
msg_throttle = Int(3,
sync=True,
help="""Maximum number of msgs the
front-end can send before receiving an idle msg from the back-end.""")
version = Int(0, sync=True, help="""Widget's version""")
keys = List()
def _keys_default(self):
return [name for name in self.traits(sync=True)]
_property_lock = Tuple((None, None))
_send_state_lock = Int(0)
_states_to_send = Set(allow_none=False)
_display_callbacks = Instance(CallbackDispatcher, ())
_msg_callbacks = Instance(CallbackDispatcher, ())
#-------------------------------------------------------------------------
# (Con/de)structor
#-------------------------------------------------------------------------
def __init__(self, **kwargs):
"""Public constructor"""
self._model_id = kwargs.pop('model_id', None)
super(Widget, self).__init__(**kwargs)
Widget._call_widget_constructed(self)
self.open()
def __del__(self):
"""Object disposal"""
self.close()
#-------------------------------------------------------------------------
# Properties
#-------------------------------------------------------------------------
def open(self):
"""Open a comm to the frontend if one isn't already open."""
if self.comm is None:
args = dict(target_name='ipython.widget',
data={
'model_name': self._model_name,
'model_module': self._model_module
})
if self._model_id is not None:
args['comm_id'] = self._model_id
self.comm = Comm(**args)
def _comm_changed(self, name, new):
"""Called when the comm is changed."""
if new is None:
return
self._model_id = self.model_id
self.comm.on_msg(self._handle_msg)
Widget.widgets[self.model_id] = self
# first update
self.send_state()
@property
def model_id(self):
"""Gets the model id of this widget.
If a Comm doesn't exist yet, a Comm will be created automagically."""
return self.comm.comm_id
#-------------------------------------------------------------------------
# Methods
#-------------------------------------------------------------------------
def close(self):
"""Close method.
Closes the underlying comm.
When the comm is closed, all of the widget views are automatically
removed from the front-end."""
if self.comm is not None:
Widget.widgets.pop(self.model_id, None)
self.comm.close()
self.comm = None
def send_state(self, key=None):
"""Sends the widget state, or a piece of it, to the front-end.
Parameters
----------
key : unicode, or iterable (optional)
A single property's name or iterable of property names to sync with the front-end.
"""
self._send({"method": "update", "state": self.get_state(key=key)})
def get_state(self, key=None):
"""Gets the widget state, or a piece of it.
Parameters
----------
key : unicode or iterable (optional)
A single property's name or iterable of property names to get.
"""
if key is None:
keys = self.keys
elif isinstance(key, string_types):
keys = [key]
elif isinstance(key, collections.Iterable):
keys = key
else:
raise ValueError(
"key must be a string, an iterable of keys, or None")
state = {}
for k in keys:
f = self.trait_metadata(k, 'to_json', self._trait_to_json)
value = getattr(self, k)
state[k] = f(value)
return state
def set_state(self, sync_data):
"""Called when a state is received from the front-end."""
for name in self.keys:
if name in sync_data:
json_value = sync_data[name]
from_json = self.trait_metadata(name, 'from_json',
self._trait_from_json)
with self._lock_property(name, json_value):
setattr(self, name, from_json(json_value))
def send(self, content):
"""Sends a custom msg to the widget model in the front-end.
Parameters
----------
content : dict
Content of the message to send.
"""
self._send({"method": "custom", "content": content})
def on_msg(self, callback, remove=False):
"""(Un)Register a custom msg receive callback.
Parameters
----------
callback: callable
callback will be passed two arguments when a message arrives::
callback(widget, content)
remove: bool
True if the callback should be unregistered."""
self._msg_callbacks.register_callback(callback, remove=remove)
def on_displayed(self, callback, remove=False):
"""(Un)Register a widget displayed callback.
Parameters
----------
callback: method handler
Must have a signature of::
callback(widget, **kwargs)
kwargs from display are passed through without modification.
remove: bool
True if the callback should be unregistered."""
self._display_callbacks.register_callback(callback, remove=remove)
#-------------------------------------------------------------------------
# Support methods
#-------------------------------------------------------------------------
@contextmanager
def _lock_property(self, key, value):
"""Lock a property-value pair.
The value should be the JSON state of the property.
NOTE: This, in addition to the single lock for all state changes, is
flawed. In the future we may want to look into buffering state changes
back to the front-end."""
self._property_lock = (key, value)
try:
yield
finally:
self._property_lock = (None, None)
@contextmanager
def hold_sync(self):
"""Hold syncing any state until the context manager is released"""
# We increment a value so that this can be nested. Syncing will happen when
# all levels have been released.
self._send_state_lock += 1
try:
yield
finally:
self._send_state_lock -= 1
if self._send_state_lock == 0:
self.send_state(self._states_to_send)
self._states_to_send.clear()
def _should_send_property(self, key, value):
"""Check the property lock (property_lock)"""
to_json = self.trait_metadata(key, 'to_json', self._trait_to_json)
if (key == self._property_lock[0]
and to_json(value) == self._property_lock[1]):
return False
elif self._send_state_lock > 0:
self._states_to_send.add(key)
return False
else:
return True
# Event handlers
@_show_traceback
def _handle_msg(self, msg):
"""Called when a msg is received from the front-end"""
data = msg['content']['data']
method = data['method']
# Handle backbone sync methods CREATE, PATCH, and UPDATE all in one.
if method == 'backbone':
if 'sync_data' in data:
sync_data = data['sync_data']
self.set_state(sync_data) # handles all methods
# Handle a state request.
elif method == 'request_state':
self.send_state()
# Handle a custom msg from the front-end.
elif method == 'custom':
if 'content' in data:
self._handle_custom_msg(data['content'])
# Catch remainder.
else:
self.log.error(
'Unknown front-end to back-end widget msg with method "%s"' %
method)
def _handle_custom_msg(self, content):
"""Called when a custom msg is received."""
self._msg_callbacks(self, content)
def _notify_trait(self, name, old_value, new_value):
"""Called when a property has been changed."""
# Trigger default traitlet callback machinery. This allows any user
# registered validation to be processed prior to allowing the widget
# machinery to handle the state.
LoggingConfigurable._notify_trait(self, name, old_value, new_value)
# Send the state after the user registered callbacks for trait changes
# have all fired (allows for user to validate values).
if self.comm is not None and name in self.keys:
# Make sure this isn't information that the front-end just sent us.
if self._should_send_property(name, new_value):
# Send new state to front-end
self.send_state(key=name)
def _handle_displayed(self, **kwargs):
"""Called when a view has been displayed for this widget instance"""
self._display_callbacks(self, **kwargs)
def _trait_to_json(self, x):
"""Convert a trait value to json
Traverse lists/tuples and dicts and serialize their values as well.
Replace any widgets with their model_id
"""
if isinstance(x, dict):
return {k: self._trait_to_json(v) for k, v in x.items()}
elif isinstance(x, (list, tuple)):
return [self._trait_to_json(v) for v in x]
elif isinstance(x, Widget):
return "IPY_MODEL_" + x.model_id
else:
return x # Value must be JSON-able
def _trait_from_json(self, x):
"""Convert json values to objects
Replace any strings representing valid model id values to Widget references.
"""
if isinstance(x, dict):
return {k: self._trait_from_json(v) for k, v in x.items()}
elif isinstance(x, (list, tuple)):
return [self._trait_from_json(v) for v in x]
elif isinstance(x, string_types) and x.startswith(
'IPY_MODEL_') and x[10:] in Widget.widgets:
# we want to support having child widgets at any level in a hierarchy
# trusting that a widget UUID will not appear out in the wild
return Widget.widgets[x[10:]]
else:
return x
def _ipython_display_(self, **kwargs):
"""Called when `IPython.display.display` is called on the widget."""
# Show view.
if self._view_name is not None:
self._send({"method": "display"})
self._handle_displayed(**kwargs)
def _send(self, msg):
"""Sends a message to the model in the front-end."""
self.comm.send(msg)
class TaskScheduler(SessionFactory):
"""Python TaskScheduler object.
This is the simplest object that supports msg_id based
DAG dependencies. *Only* task msg_ids are checked, not
msg_ids of jobs submitted via the MUX queue.
"""
hwm = Integer(1,
config=True,
help="""specify the High Water Mark (HWM) for the downstream
socket in the Task scheduler. This is the maximum number
of allowed outstanding tasks on each engine.
The default (1) means that only one task can be outstanding on each
engine. Setting TaskScheduler.hwm=0 means there is no limit, and the
engines continue to be assigned tasks while they are working,
effectively hiding network latency behind computation, but can result
in an imbalance of work when submitting many heterogenous tasks all at
once. Any positive value greater than one is a compromise between the
two.
""")
scheme_name = Enum(
('leastload', 'pure', 'lru', 'plainrandom', 'weighted', 'twobin'),
'leastload',
config=True,
allow_none=False,
help="""select the task scheduler scheme [default: Python LRU]
Options are: 'pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'"""
)
def _scheme_name_changed(self, old, new):
self.log.debug("Using scheme %r" % new)
self.scheme = globals()[new]
# input arguments:
scheme = Instance(FunctionType) # function for determining the destination
def _scheme_default(self):
return leastload
client_stream = Instance(zmqstream.ZMQStream) # client-facing stream
engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream
notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream
mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream
query_stream = Instance(zmqstream.ZMQStream) # hub-facing DEALER stream
# internals:
queue = Instance(deque) # sorted list of Jobs
def _queue_default(self):
return deque()
queue_map = Dict() # dict by msg_id of Jobs (for O(1) access to the Queue)
graph = Dict() # dict by msg_id of [ msg_ids that depend on key ]
retries = Dict() # dict by msg_id of retries remaining (non-neg ints)
# waiting = List() # list of msg_ids ready to run, but haven't due to HWM
pending = Dict() # dict by engine_uuid of submitted tasks
completed = Dict() # dict by engine_uuid of completed tasks
failed = Dict() # dict by engine_uuid of failed tasks
destinations = Dict(
) # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed)
clients = Dict() # dict by msg_id for who submitted the task
targets = List() # list of target IDENTs
loads = List() # list of engine loads
# full = Set() # set of IDENTs that have HWM outstanding tasks
all_completed = Set() # set of all completed tasks
all_failed = Set() # set of all failed tasks
all_done = Set() # set of all finished tasks=union(completed,failed)
all_ids = Set() # set of all submitted task IDs
ident = CBytes() # ZMQ identity. This should just be self.session.session
# but ensure Bytes
def _ident_default(self):
return self.session.bsession
def start(self):
self.query_stream.on_recv(self.dispatch_query_reply)
self.session.send(self.query_stream, "connection_request", {})
self.engine_stream.on_recv(self.dispatch_result, copy=False)
self.client_stream.on_recv(self.dispatch_submission, copy=False)
self._notification_handlers = dict(
registration_notification=self._register_engine,
unregistration_notification=self._unregister_engine)
self.notifier_stream.on_recv(self.dispatch_notification)
self.log.info("Scheduler started [%s]" % self.scheme_name)
def resume_receiving(self):
"""Resume accepting jobs."""
self.client_stream.on_recv(self.dispatch_submission, copy=False)
def stop_receiving(self):
"""Stop accepting jobs while there are no engines.
Leave them in the ZMQ queue."""
self.client_stream.on_recv(None)
#-----------------------------------------------------------------------
# [Un]Registration Handling
#-----------------------------------------------------------------------
def dispatch_query_reply(self, msg):
"""handle reply to our initial connection request"""
try:
idents, msg = self.session.feed_identities(msg)
except ValueError:
self.log.warn("task::Invalid Message: %r", msg)
return
try:
msg = self.session.unserialize(msg)
except ValueError:
self.log.warn("task::Unauthorized message from: %r" % idents)
return
content = msg['content']
for uuid in content.get('engines', {}).values():
self._register_engine(cast_bytes(uuid))
@util.log_errors
def dispatch_notification(self, msg):
"""dispatch register/unregister events."""
try:
idents, msg = self.session.feed_identities(msg)
except ValueError:
self.log.warn("task::Invalid Message: %r", msg)
return
try:
msg = self.session.unserialize(msg)
except ValueError:
self.log.warn("task::Unauthorized message from: %r" % idents)
return
msg_type = msg['header']['msg_type']
handler = self._notification_handlers.get(msg_type, None)
if handler is None:
self.log.error("Unhandled message type: %r" % msg_type)
else:
try:
handler(cast_bytes(msg['content']['uuid']))
except Exception:
self.log.error("task::Invalid notification msg: %r",
msg,
exc_info=True)
def _register_engine(self, uid):
"""New engine with ident `uid` became available."""
# head of the line:
self.targets.insert(0, uid)
self.loads.insert(0, 0)
# initialize sets
self.completed[uid] = set()
self.failed[uid] = set()
self.pending[uid] = {}
# rescan the graph:
self.update_graph(None)
def _unregister_engine(self, uid):
"""Existing engine with ident `uid` became unavailable."""
if len(self.targets) == 1:
# this was our only engine
pass
# handle any potentially finished tasks:
self.engine_stream.flush()
# don't pop destinations, because they might be used later
# map(self.destinations.pop, self.completed.pop(uid))
# map(self.destinations.pop, self.failed.pop(uid))
# prevent this engine from receiving work
idx = self.targets.index(uid)
self.targets.pop(idx)
self.loads.pop(idx)
# wait 5 seconds before cleaning up pending jobs, since the results might
# still be incoming
if self.pending[uid]:
dc = ioloop.DelayedCallback(
lambda: self.handle_stranded_tasks(uid), 5000, self.loop)
dc.start()
else:
self.completed.pop(uid)
self.failed.pop(uid)
def handle_stranded_tasks(self, engine):
"""Deal with jobs resident in an engine that died."""
lost = self.pending[engine]
for msg_id in lost.keys():
if msg_id not in self.pending[engine]:
# prevent double-handling of messages
continue
raw_msg = lost[msg_id].raw_msg
idents, msg = self.session.feed_identities(raw_msg, copy=False)
parent = self.session.unpack(msg[1].bytes)
idents = [engine, idents[0]]
# build fake error reply
try:
raise error.EngineError(
"Engine %r died while running task %r" % (engine, msg_id))
except:
content = error.wrap_exception()
# build fake metadata
md = dict(
status=u'error',
engine=engine.decode('ascii'),
date=datetime.now(),
)
msg = self.session.msg('apply_reply',
content,
parent=parent,
metadata=md)
raw_reply = list(
map(zmq.Message, self.session.serialize(msg, ident=idents)))
# and dispatch it
self.dispatch_result(raw_reply)
# finally scrub completed/failed lists
self.completed.pop(engine)
self.failed.pop(engine)
#-----------------------------------------------------------------------
# Job Submission
#-----------------------------------------------------------------------
@util.log_errors
def dispatch_submission(self, raw_msg):
"""Dispatch job submission to appropriate handlers."""
# ensure targets up to date:
self.notifier_stream.flush()
try:
idents, msg = self.session.feed_identities(raw_msg, copy=False)
msg = self.session.unserialize(msg, content=False, copy=False)
except Exception:
self.log.error("task::Invaid task msg: %r" % raw_msg,
exc_info=True)
return
# send to monitor
self.mon_stream.send_multipart([b'intask'] + raw_msg, copy=False)
header = msg['header']
md = msg['metadata']
msg_id = header['msg_id']
self.all_ids.add(msg_id)
# get targets as a set of bytes objects
# from a list of unicode objects
targets = md.get('targets', [])
targets = set(map(cast_bytes, targets))
retries = md.get('retries', 0)
self.retries[msg_id] = retries
# time dependencies
after = md.get('after', None)
if after:
after = Dependency(after)
if after.all:
if after.success:
after = Dependency(
after.difference(self.all_completed),
success=after.success,
failure=after.failure,
all=after.all,
)
if after.failure:
after = Dependency(
after.difference(self.all_failed),
success=after.success,
failure=after.failure,
all=after.all,
)
if after.check(self.all_completed, self.all_failed):
# recast as empty set, if `after` already met,
# to prevent unnecessary set comparisons
after = MET
else:
after = MET
# location dependencies
follow = Dependency(md.get('follow', []))
timeout = md.get('timeout', None)
if timeout:
timeout = float(timeout)
job = Job(
msg_id=msg_id,
raw_msg=raw_msg,
idents=idents,
msg=msg,
header=header,
targets=targets,
after=after,
follow=follow,
timeout=timeout,
metadata=md,
)
# validate and reduce dependencies:
for dep in after, follow:
if not dep: # empty dependency
continue
# check valid:
if msg_id in dep or dep.difference(self.all_ids):
self.queue_map[msg_id] = job
return self.fail_unreachable(msg_id, error.InvalidDependency)
# check if unreachable:
if dep.unreachable(self.all_completed, self.all_failed):
self.queue_map[msg_id] = job
return self.fail_unreachable(msg_id)
if after.check(self.all_completed, self.all_failed):
# time deps already met, try to run
if not self.maybe_run(job):
# can't run yet
if msg_id not in self.all_failed:
# could have failed as unreachable
self.save_unmet(job)
else:
self.save_unmet(job)
def job_timeout(self, job, timeout_id):
"""callback for a job's timeout.
The job may or may not have been run at this point.
"""
if job.timeout_id != timeout_id:
# not the most recent call
return
now = time.time()
if job.timeout >= (now + 1):
self.log.warn("task %s timeout fired prematurely: %s > %s",
job.msg_id, job.timeout, now)
if job.msg_id in self.queue_map:
# still waiting, but ran out of time
self.log.info("task %r timed out", job.msg_id)
self.fail_unreachable(job.msg_id, error.TaskTimeout)
def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
"""a task has become unreachable, send a reply with an ImpossibleDependency
error."""
if msg_id not in self.queue_map:
self.log.error("task %r already failed!", msg_id)
return
job = self.queue_map.pop(msg_id)
# lazy-delete from the queue
job.removed = True
for mid in job.dependents:
if mid in self.graph:
self.graph[mid].remove(msg_id)
try:
raise why()
except:
content = error.wrap_exception()
self.log.debug("task %r failing as unreachable with: %s", msg_id,
content['ename'])
self.all_done.add(msg_id)
self.all_failed.add(msg_id)
msg = self.session.send(self.client_stream,
'apply_reply',
content,
parent=job.header,
ident=job.idents)
self.session.send(self.mon_stream,
msg,
ident=[b'outtask'] + job.idents)
self.update_graph(msg_id, success=False)
def available_engines(self):
"""return a list of available engine indices based on HWM"""
if not self.hwm:
return list(range(len(self.targets)))
available = []
for idx in range(len(self.targets)):
if self.loads[idx] < self.hwm:
available.append(idx)
return available
def maybe_run(self, job):
"""check location dependencies, and run if they are met."""
msg_id = job.msg_id
self.log.debug("Attempting to assign task %s", msg_id)
available = self.available_engines()
if not available:
# no engines, definitely can't run
return False
if job.follow or job.targets or job.blacklist or self.hwm:
# we need a can_run filter
def can_run(idx):
# check hwm
if self.hwm and self.loads[idx] == self.hwm:
return False
target = self.targets[idx]
# check blacklist
if target in job.blacklist:
return False
# check targets
if job.targets and target not in job.targets:
return False
# check follow
return job.follow.check(self.completed[target],
self.failed[target])
indices = list(filter(can_run, available))
if not indices:
# couldn't run
if job.follow.all:
# check follow for impossibility
dests = set()
relevant = set()
if job.follow.success:
relevant = self.all_completed
if job.follow.failure:
relevant = relevant.union(self.all_failed)
for m in job.follow.intersection(relevant):
dests.add(self.destinations[m])
if len(dests) > 1:
self.queue_map[msg_id] = job
self.fail_unreachable(msg_id)
return False
if job.targets:
# check blacklist+targets for impossibility
job.targets.difference_update(job.blacklist)
if not job.targets or not job.targets.intersection(
self.targets):
self.queue_map[msg_id] = job
self.fail_unreachable(msg_id)
return False
return False
else:
indices = None
self.submit_task(job, indices)
return True
def save_unmet(self, job):
"""Save a message for later submission when its dependencies are met."""
msg_id = job.msg_id
self.log.debug("Adding task %s to the queue", msg_id)
self.queue_map[msg_id] = job
self.queue.append(job)
# track the ids in follow or after, but not those already finished
for dep_id in job.after.union(job.follow).difference(self.all_done):
if dep_id not in self.graph:
self.graph[dep_id] = set()
self.graph[dep_id].add(msg_id)
# schedule timeout callback
if job.timeout:
timeout_id = job.timeout_id = job.timeout_id + 1
self.loop.add_timeout(time.time() + job.timeout,
lambda: self.job_timeout(job, timeout_id))
def submit_task(self, job, indices=None):
"""Submit a task to any of a subset of our targets."""
if indices:
loads = [self.loads[i] for i in indices]
else:
loads = self.loads
idx = self.scheme(loads)
if indices:
idx = indices[idx]
target = self.targets[idx]
# print (target, map(str, msg[:3]))
# send job to the engine
self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
self.engine_stream.send_multipart(job.raw_msg, copy=False)
# update load
self.add_job(idx)
self.pending[target][job.msg_id] = job
# notify Hub
content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii'))
self.session.send(self.mon_stream,
'task_destination',
content=content,
ident=[b'tracktask', self.ident])
#-----------------------------------------------------------------------
# Result Handling
#-----------------------------------------------------------------------
@util.log_errors
def dispatch_result(self, raw_msg):
"""dispatch method for result replies"""
try:
idents, msg = self.session.feed_identities(raw_msg, copy=False)
msg = self.session.unserialize(msg, content=False, copy=False)
engine = idents[0]
try:
idx = self.targets.index(engine)
except ValueError:
pass # skip load-update for dead engines
else:
self.finish_job(idx)
except Exception:
self.log.error("task::Invalid result: %r", raw_msg, exc_info=True)
return
md = msg['metadata']
parent = msg['parent_header']
if md.get('dependencies_met', True):
success = (md['status'] == 'ok')
msg_id = parent['msg_id']
retries = self.retries[msg_id]
if not success and retries > 0:
# failed
self.retries[msg_id] = retries - 1
self.handle_unmet_dependency(idents, parent)
else:
del self.retries[msg_id]
# relay to client and update graph
self.handle_result(idents, parent, raw_msg, success)
# send to Hub monitor
self.mon_stream.send_multipart([b'outtask'] + raw_msg,
copy=False)
else:
self.handle_unmet_dependency(idents, parent)
def handle_result(self, idents, parent, raw_msg, success=True):
"""handle a real task result, either success or failure"""
# first, relay result to client
engine = idents[0]
client = idents[1]
# swap_ids for ROUTER-ROUTER mirror
raw_msg[:2] = [client, engine]
# print (map(str, raw_msg[:4]))
self.client_stream.send_multipart(raw_msg, copy=False)
# now, update our data structures
msg_id = parent['msg_id']
self.pending[engine].pop(msg_id)
if success:
self.completed[engine].add(msg_id)
self.all_completed.add(msg_id)
else:
self.failed[engine].add(msg_id)
self.all_failed.add(msg_id)
self.all_done.add(msg_id)
self.destinations[msg_id] = engine
self.update_graph(msg_id, success)
def handle_unmet_dependency(self, idents, parent):
"""handle an unmet dependency"""
engine = idents[0]
msg_id = parent['msg_id']
job = self.pending[engine].pop(msg_id)
job.blacklist.add(engine)
if job.blacklist == job.targets:
self.queue_map[msg_id] = job
self.fail_unreachable(msg_id)
elif not self.maybe_run(job):
# resubmit failed
if msg_id not in self.all_failed:
# put it back in our dependency tree
self.save_unmet(job)
if self.hwm:
try:
idx = self.targets.index(engine)
except ValueError:
pass # skip load-update for dead engines
else:
if self.loads[idx] == self.hwm - 1:
self.update_graph(None)
def update_graph(self, dep_id=None, success=True):
"""dep_id just finished. Update our dependency
graph and submit any jobs that just became runnable.
Called with dep_id=None to update entire graph for hwm, but without finishing a task.
"""
# print ("\n\n***********")
# pprint (dep_id)
# pprint (self.graph)
# pprint (self.queue_map)
# pprint (self.all_completed)
# pprint (self.all_failed)
# print ("\n\n***********\n\n")
# update any jobs that depended on the dependency
msg_ids = self.graph.pop(dep_id, [])
# recheck *all* jobs if
# a) we have HWM and an engine just become no longer full
# or b) dep_id was given as None
if dep_id is None or self.hwm and any(
[load == self.hwm - 1 for load in self.loads]):
jobs = self.queue
using_queue = True
else:
using_queue = False
jobs = deque(sorted(self.queue_map[msg_id] for msg_id in msg_ids))
to_restore = []
while jobs:
job = jobs.popleft()
if job.removed:
continue
msg_id = job.msg_id
put_it_back = True
if job.after.unreachable(self.all_completed, self.all_failed)\
or job.follow.unreachable(self.all_completed, self.all_failed):
self.fail_unreachable(msg_id)
put_it_back = False
elif job.after.check(self.all_completed,
self.all_failed): # time deps met, maybe run
if self.maybe_run(job):
put_it_back = False
self.queue_map.pop(msg_id)
for mid in job.dependents:
if mid in self.graph:
self.graph[mid].remove(msg_id)
# abort the loop if we just filled up all of our engines.
# avoids an O(N) operation in situation of full queue,
# where graph update is triggered as soon as an engine becomes
# non-full, and all tasks after the first are checked,
# even though they can't run.
if not self.available_engines():
break
if using_queue and put_it_back:
# popped a job from the queue but it neither ran nor failed,
# so we need to put it back when we are done
# make sure to_restore preserves the same ordering
to_restore.append(job)
# put back any tasks we popped but didn't run
if using_queue:
self.queue.extendleft(to_restore)
#----------------------------------------------------------------------
# methods to be overridden by subclasses
#----------------------------------------------------------------------
def add_job(self, idx):
"""Called after self.targets[idx] just got the job with header.
Override with subclasses. The default ordering is simple LRU.
The default loads are the number of outstanding jobs."""
self.loads[idx] += 1
for lis in (self.targets, self.loads):
lis.append(lis.pop(idx))
def finish_job(self, idx):
"""Called after self.targets[idx] just finished a job.
Override with subclasses."""
self.loads[idx] -= 1
class Exporter(LoggingConfigurable):
"""
Class containing methods that sequentially run a list of preprocessors on a
NotebookNode object and then return the modified NotebookNode object and
accompanying resources dict.
"""
file_extension = Unicode(
'txt', config=True,
help="Extension of the file that should be written to disk"
)
# MIME type of the result file, for HTTP response headers.
# This is *not* a traitlet, because we want to be able to access it from
# the class, not just on instances.
output_mimetype = ''
#Configurability, allows the user to easily add filters and preprocessors.
preprocessors = List(config=True,
help="""List of preprocessors, by name or namespace, to enable.""")
_preprocessors = List()
default_preprocessors = List(['IPython.nbconvert.preprocessors.coalesce_streams',
'IPython.nbconvert.preprocessors.SVG2PDFPreprocessor',
'IPython.nbconvert.preprocessors.ExtractOutputPreprocessor',
'IPython.nbconvert.preprocessors.CSSHTMLHeaderPreprocessor',
'IPython.nbconvert.preprocessors.RevealHelpPreprocessor',
'IPython.nbconvert.preprocessors.LatexPreprocessor',
'IPython.nbconvert.preprocessors.ClearOutputPreprocessor',
'IPython.nbconvert.preprocessors.ExecutePreprocessor',
'IPython.nbconvert.preprocessors.HighlightMagicsPreprocessor'],
config=True,
help="""List of preprocessors available by default, by name, namespace,
instance, or type.""")
def __init__(self, config=None, **kw):
"""
Public constructor
Parameters
----------
config : config
User configuration instance.
"""
with_default_config = self.default_config
if config:
with_default_config.merge(config)
super(Exporter, self).__init__(config=with_default_config, **kw)
self._init_preprocessors()
@property
def default_config(self):
return Config()
def from_notebook_node(self, nb, resources=None, **kw):
"""
Convert a notebook from a notebook node instance.
Parameters
----------
nb : :class:`~IPython.nbformat.current.NotebookNode`
Notebook node
resources : dict
Additional resources that can be accessed read/write by
preprocessors and filters.
**kw
Ignored (?)
"""
nb_copy = copy.deepcopy(nb)
resources = self._init_resources(resources)
if 'language' in nb['metadata']:
resources['language'] = nb['metadata']['language'].lower()
# Preprocess
nb_copy, resources = self._preprocess(nb_copy, resources)
return nb_copy, resources
def from_filename(self, filename, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
filename : str
Full filename of the notebook file to open and convert.
"""
# Pull the metadata from the filesystem.
if resources is None:
resources = ResourcesDict()
if not 'metadata' in resources or resources['metadata'] == '':
resources['metadata'] = ResourcesDict()
basename = os.path.basename(filename)
notebook_name = basename[:basename.rfind('.')]
resources['metadata']['name'] = notebook_name
modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
resources['metadata']['modified_date'] = modified_date.strftime(text.date_format)
with io.open(filename, encoding='utf-8') as f:
return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw)
def from_file(self, file_stream, resources=None, **kw):
"""
Convert a notebook from a notebook file.
Parameters
----------
file_stream : file-like object
Notebook file-like object to convert.
"""
return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw)
def register_preprocessor(self, preprocessor, enabled=False):
"""
Register a preprocessor.
Preprocessors are classes that act upon the notebook before it is
passed into the Jinja templating engine. preprocessors are also
capable of passing additional information to the Jinja
templating engine.
Parameters
----------
preprocessor : preprocessor
"""
if preprocessor is None:
raise TypeError('preprocessor')
isclass = isinstance(preprocessor, type)
constructed = not isclass
# Handle preprocessor's registration based on it's type
if constructed and isinstance(preprocessor, py3compat.string_types):
# Preprocessor is a string, import the namespace and recursively call
# this register_preprocessor method
preprocessor_cls = import_item(preprocessor)
return self.register_preprocessor(preprocessor_cls, enabled)
if constructed and hasattr(preprocessor, '__call__'):
# Preprocessor is a function, no need to construct it.
# Register and return the preprocessor.
if enabled:
preprocessor.enabled = True
self._preprocessors.append(preprocessor)
return preprocessor
elif isclass and isinstance(preprocessor, MetaHasTraits):
# Preprocessor is configurable. Make sure to pass in new default for
# the enabled flag if one was specified.
self.register_preprocessor(preprocessor(parent=self), enabled)
elif isclass:
# Preprocessor is not configurable, construct it
self.register_preprocessor(preprocessor(), enabled)
else:
# Preprocessor is an instance of something without a __call__
# attribute.
raise TypeError('preprocessor')
def _init_preprocessors(self):
"""
Register all of the preprocessors needed for this exporter, disabled
unless specified explicitly.
"""
self._preprocessors = []
# Load default preprocessors (not necessarly enabled by default).
for preprocessor in self.default_preprocessors:
self.register_preprocessor(preprocessor)
# Load user-specified preprocessors. Enable by default.
for preprocessor in self.preprocessors:
self.register_preprocessor(preprocessor, enabled=True)
def _init_resources(self, resources):
#Make sure the resources dict is of ResourcesDict type.
if resources is None:
resources = ResourcesDict()
if not isinstance(resources, ResourcesDict):
new_resources = ResourcesDict()
new_resources.update(resources)
resources = new_resources
#Make sure the metadata extension exists in resources
if 'metadata' in resources:
if not isinstance(resources['metadata'], ResourcesDict):
resources['metadata'] = ResourcesDict(resources['metadata'])
else:
resources['metadata'] = ResourcesDict()
if not resources['metadata']['name']:
resources['metadata']['name'] = 'Notebook'
#Set the output extension
resources['output_extension'] = self.file_extension
return resources
def _preprocess(self, nb, resources):
"""
Preprocess the notebook before passing it into the Jinja engine.
To preprocess the notebook is to apply all of the
Parameters
----------
nb : notebook node
notebook that is being exported.
resources : a dict of additional resources that
can be accessed read/write by preprocessors
"""
# Do a copy.deepcopy first,
# we are never safe enough with what the preprocessors could do.
nbc = copy.deepcopy(nb)
resc = copy.deepcopy(resources)
#Run each preprocessor on the notebook. Carry the output along
#to each preprocessor
for preprocessor in self._preprocessors:
nbc, resc = preprocessor(nbc, resc)
return nbc, resc
class View(HasTraits):
"""Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
Don't use this class, use subclasses.
Methods
-------
spin
flushes incoming results and registration state changes
control methods spin, and requesting `ids` also ensures up to date
wait
wait on one or more msg_ids
execution methods
apply
legacy: execute, run
data movement
push, pull, scatter, gather
query methods
get_result, queue_status, purge_results, result_status
control methods
abort, shutdown
"""
# flags
block = Bool(False)
track = Bool(True)
targets = Any()
history = List()
outstanding = Set()
results = Dict()
client = Instance('IPython.parallel.Client')
_socket = Instance('zmq.Socket')
_flag_names = List(['targets', 'block', 'track'])
_targets = Any()
_idents = Any()
def __init__(self, client=None, socket=None, **flags):
super(View, self).__init__(client=client, _socket=socket)
self.block = client.block
self.set_flags(**flags)
assert not self.__class__ is View, "Don't use base View objects, use subclasses"
def __repr__(self):
strtargets = str(self.targets)
if len(strtargets) > 16:
strtargets = strtargets[:12] + '...]'
return "<%s %s>" % (self.__class__.__name__, strtargets)
def __len__(self):
if isinstance(self.targets, list):
return len(self.targets)
elif isinstance(self.targets, int):
return 1
else:
return len(self.client)
def set_flags(self, **kwargs):
"""set my attribute flags by keyword.
Views determine behavior with a few attributes (`block`, `track`, etc.).
These attributes can be set all at once by name with this method.
Parameters
----------
block : bool
whether to wait for results
track : bool
whether to create a MessageTracker to allow the user to
safely edit after arrays and buffers during non-copying
sends.
"""
for name, value in kwargs.iteritems():
if name not in self._flag_names:
raise KeyError("Invalid name: %r" % name)
else:
setattr(self, name, value)
@contextmanager
def temp_flags(self, **kwargs):
"""temporarily set flags, for use in `with` statements.
See set_flags for permanent setting of flags
Examples
--------
>>> view.track=False
...
>>> with view.temp_flags(track=True):
... ar = view.apply(dostuff, my_big_array)
... ar.tracker.wait() # wait for send to finish
>>> view.track
False
"""
# preflight: save flags, and set temporaries
saved_flags = {}
for f in self._flag_names:
saved_flags[f] = getattr(self, f)
self.set_flags(**kwargs)
# yield to the with-statement block
try:
yield
finally:
# postflight: restore saved flags
self.set_flags(**saved_flags)
#----------------------------------------------------------------
# apply
#----------------------------------------------------------------
@sync_results
@save_ids
def _really_apply(self, f, args, kwargs, block=None, **options):
"""wrapper for client.send_apply_request"""
raise NotImplementedError("Implement in subclasses")
def apply(self, f, *args, **kwargs):
"""calls f(*args, **kwargs) on remote engines, returning the result.
This method sets all apply flags via this View's attributes.
if self.block is False:
returns AsyncResult
else:
returns actual result of f(*args, **kwargs)
"""
return self._really_apply(f, args, kwargs)
def apply_async(self, f, *args, **kwargs):
"""calls f(*args, **kwargs) on remote engines in a nonblocking manner.
returns AsyncResult
"""
return self._really_apply(f, args, kwargs, block=False)
@spin_after
def apply_sync(self, f, *args, **kwargs):
"""calls f(*args, **kwargs) on remote engines in a blocking manner,
returning the result.
returns: actual result of f(*args, **kwargs)
"""
return self._really_apply(f, args, kwargs, block=True)
#----------------------------------------------------------------
# wrappers for client and control methods
#----------------------------------------------------------------
@sync_results
def spin(self):
"""spin the client, and sync"""
self.client.spin()
@sync_results
def wait(self, jobs=None, timeout=-1):
"""waits on one or more `jobs`, for up to `timeout` seconds.
Parameters
----------
jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects
ints are indices to self.history
strs are msg_ids
default: wait on all outstanding messages
timeout : float
a time in seconds, after which to give up.
default is -1, which means no timeout
Returns
-------
True : when all msg_ids are done
False : timeout reached, some msg_ids still outstanding
"""
if jobs is None:
jobs = self.history
return self.client.wait(jobs, timeout)
def abort(self, jobs=None, targets=None, block=None):
"""Abort jobs on my engines.
Parameters
----------
jobs : None, str, list of strs, optional
if None: abort all jobs.
else: abort specific msg_id(s).
"""
block = block if block is not None else self.block
targets = targets if targets is not None else self.targets
jobs = jobs if jobs is not None else list(self.outstanding)
return self.client.abort(jobs=jobs, targets=targets, block=block)
def queue_status(self, targets=None, verbose=False):
"""Fetch the Queue status of my engines"""
targets = targets if targets is not None else self.targets
return self.client.queue_status(targets=targets, verbose=verbose)
def purge_results(self, jobs=[], targets=[]):
"""Instruct the controller to forget specific results."""
if targets is None or targets == 'all':
targets = self.targets
return self.client.purge_results(jobs=jobs, targets=targets)
def shutdown(self, targets=None, restart=False, hub=False, block=None):
"""Terminates one or more engine processes, optionally including the hub.
"""
block = self.block if block is None else block
if targets is None or targets == 'all':
targets = self.targets
return self.client.shutdown(targets=targets,
restart=restart,
hub=hub,
block=block)
@spin_after
def get_result(self, indices_or_msg_ids=None):
"""return one or more results, specified by history index or msg_id.
See client.get_result for details.
"""
if indices_or_msg_ids is None:
indices_or_msg_ids = -1
if isinstance(indices_or_msg_ids, int):
indices_or_msg_ids = self.history[indices_or_msg_ids]
elif isinstance(indices_or_msg_ids, (list, tuple, set)):
indices_or_msg_ids = list(indices_or_msg_ids)
for i, index in enumerate(indices_or_msg_ids):
if isinstance(index, int):
indices_or_msg_ids[i] = self.history[index]
return self.client.get_result(indices_or_msg_ids)
#-------------------------------------------------------------------
# Map
#-------------------------------------------------------------------
def map(self, f, *sequences, **kwargs):
"""override in subclasses"""
raise NotImplementedError
def map_async(self, f, *sequences, **kwargs):
"""Parallel version of builtin `map`, using this view's engines.
This is equivalent to map(...block=False)
See `self.map` for details.
"""
if 'block' in kwargs:
raise TypeError(
"map_async doesn't take a `block` keyword argument.")
kwargs['block'] = False
return self.map(f, *sequences, **kwargs)
def map_sync(self, f, *sequences, **kwargs):
"""Parallel version of builtin `map`, using this view's engines.
This is equivalent to map(...block=True)
See `self.map` for details.
"""
if 'block' in kwargs:
raise TypeError(
"map_sync doesn't take a `block` keyword argument.")
kwargs['block'] = True
return self.map(f, *sequences, **kwargs)
def imap(self, f, *sequences, **kwargs):
"""Parallel version of `itertools.imap`.
See `self.map` for details.
"""
return iter(self.map_async(f, *sequences, **kwargs))
#-------------------------------------------------------------------
# Decorators
#-------------------------------------------------------------------
def remote(self, block=True, **flags):
"""Decorator for making a RemoteFunction"""
block = self.block if block is None else block
return remote(self, block=block, **flags)
def parallel(self, dist='b', block=None, **flags):
"""Decorator for making a ParallelFunction"""
block = self.block if block is None else block
return parallel(self, dist=dist, block=block, **flags)
class LazyConfigValue(HasTraits):
"""Proxy object for exposing methods on configurable containers
Exposes:
- append, extend, insert on lists
- update on dicts
- update, add on sets
"""
_value = None
# list methods
_extend = List()
_prepend = List()
def append(self, obj):
self._extend.append(obj)
def extend(self, other):
self._extend.extend(other)
def prepend(self, other):
"""like list.extend, but for the front"""
self._prepend[:0] = other
_inserts = List()
def insert(self, index, other):
if not isinstance(index, int):
raise TypeError("An integer is required")
self._inserts.append((index, other))
# dict methods
# update is used for both dict and set
_update = Any()
def update(self, other):
if self._update is None:
if isinstance(other, dict):
self._update = {}
else:
self._update = set()
self._update.update(other)
# set methods
def add(self, obj):
self.update({obj})
def get_value(self, initial):
"""construct the value from the initial one
after applying any insert / extend / update changes
"""
if self._value is not None:
return self._value
value = copy.deepcopy(initial)
if isinstance(value, list):
for idx, obj in self._inserts:
value.insert(idx, obj)
value[:0] = self._prepend
value.extend(self._extend)
elif isinstance(value, dict):
if self._update:
value.update(self._update)
elif isinstance(value, set):
if self._update:
value.update(self._update)
self._value = value
return value
def to_dict(self):
"""return JSONable dict form of my data
Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
"""
d = {}
if self._update:
d['update'] = self._update
if self._extend:
d['extend'] = self._extend
if self._prepend:
d['prepend'] = self._prepend
elif self._inserts:
d['inserts'] = self._inserts
return d
class LoadBalancedView(View):
"""An load-balancing View that only executes via the Task scheduler.
Load-balanced views can be created with the client's `view` method:
>>> v = client.load_balanced_view()
or targets can be specified, to restrict the potential destinations:
>>> v = client.client.load_balanced_view([1,3])
which would restrict loadbalancing to between engines 1 and 3.
"""
follow = Any()
after = Any()
timeout = CFloat()
retries = Integer(0)
_task_scheme = Any()
_flag_names = List(
['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries'])
def __init__(self, client=None, socket=None, **flags):
super(LoadBalancedView, self).__init__(client=client,
socket=socket,
**flags)
self._task_scheme = client._task_scheme
def _validate_dependency(self, dep):
"""validate a dependency.
For use in `set_flags`.
"""
if dep is None or isinstance(dep,
(basestring, AsyncResult, Dependency)):
return True
elif isinstance(dep, (list, set, tuple)):
for d in dep:
if not isinstance(d, (basestring, AsyncResult)):
return False
elif isinstance(dep, dict):
if set(dep.keys()) != set(Dependency().as_dict().keys()):
return False
if not isinstance(dep['msg_ids'], list):
return False
for d in dep['msg_ids']:
if not isinstance(d, basestring):
return False
else:
return False
return True
def _render_dependency(self, dep):
"""helper for building jsonable dependencies from various input forms."""
if isinstance(dep, Dependency):
return dep.as_dict()
elif isinstance(dep, AsyncResult):
return dep.msg_ids
elif dep is None:
return []
else:
# pass to Dependency constructor
return list(Dependency(dep))
def set_flags(self, **kwargs):
"""set my attribute flags by keyword.
A View is a wrapper for the Client's apply method, but with attributes
that specify keyword arguments, those attributes can be set by keyword
argument with this method.
Parameters
----------
block : bool
whether to wait for results
track : bool
whether to create a MessageTracker to allow the user to
safely edit after arrays and buffers during non-copying
sends.
after : Dependency or collection of msg_ids
Only for load-balanced execution (targets=None)
Specify a list of msg_ids as a time-based dependency.
This job will only be run *after* the dependencies
have been met.
follow : Dependency or collection of msg_ids
Only for load-balanced execution (targets=None)
Specify a list of msg_ids as a location-based dependency.
This job will only be run on an engine where this dependency
is met.
timeout : float/int or None
Only for load-balanced execution (targets=None)
Specify an amount of time (in seconds) for the scheduler to
wait for dependencies to be met before failing with a
DependencyTimeout.
retries : int
Number of times a task will be retried on failure.
"""
super(LoadBalancedView, self).set_flags(**kwargs)
for name in ('follow', 'after'):
if name in kwargs:
value = kwargs[name]
if self._validate_dependency(value):
setattr(self, name, value)
else:
raise ValueError("Invalid dependency: %r" % value)
if 'timeout' in kwargs:
t = kwargs['timeout']
if not isinstance(t, (int, long, float, type(None))):
raise TypeError("Invalid type for timeout: %r" % type(t))
if t is not None:
if t < 0:
raise ValueError("Invalid timeout: %s" % t)
self.timeout = t
@sync_results
@save_ids
def _really_apply(self,
f,
args=None,
kwargs=None,
block=None,
track=None,
after=None,
follow=None,
timeout=None,
targets=None,
retries=None):
"""calls f(*args, **kwargs) on a remote engine, returning the result.
This method temporarily sets all of `apply`'s flags for a single call.
Parameters
----------
f : callable
args : list [default: empty]
kwargs : dict [default: empty]
block : bool [default: self.block]
whether to block
track : bool [default: self.track]
whether to ask zmq to track the message, for safe non-copying sends
!!!!!! TODO: THE REST HERE !!!!
Returns
-------
if self.block is False:
returns AsyncResult
else:
returns actual result of f(*args, **kwargs) on the engine(s)
This will be a list of self.targets is also a list (even length 1), or
the single result if self.targets is an integer engine id
"""
# validate whether we can run
if self._socket.closed:
msg = "Task farming is disabled"
if self._task_scheme == 'pure':
msg += " because the pure ZMQ scheduler cannot handle"
msg += " disappearing engines."
raise RuntimeError(msg)
if self._task_scheme == 'pure':
# pure zmq scheme doesn't support extra features
msg = "Pure ZMQ scheduler doesn't support the following flags:"
"follow, after, retries, targets, timeout"
if (follow or after or retries or targets or timeout):
# hard fail on Scheduler flags
raise RuntimeError(msg)
if isinstance(f, dependent):
# soft warn on functional dependencies
warnings.warn(msg, RuntimeWarning)
# build args
args = [] if args is None else args
kwargs = {} if kwargs is None else kwargs
block = self.block if block is None else block
track = self.track if track is None else track
after = self.after if after is None else after
retries = self.retries if retries is None else retries
follow = self.follow if follow is None else follow
timeout = self.timeout if timeout is None else timeout
targets = self.targets if targets is None else targets
if not isinstance(retries, int):
raise TypeError('retries must be int, not %r' % type(retries))
if targets is None:
idents = []
else:
idents = self.client._build_targets(targets)[0]
# ensure *not* bytes
idents = [ident.decode() for ident in idents]
after = self._render_dependency(after)
follow = self._render_dependency(follow)
subheader = dict(after=after,
follow=follow,
timeout=timeout,
targets=idents,
retries=retries)
msg = self.client.send_apply_request(self._socket,
f,
args,
kwargs,
track=track,
subheader=subheader)
tracker = None if track is False else msg['tracker']
ar = AsyncResult(self.client,
msg['header']['msg_id'],
fname=getname(f),
targets=None,
tracker=tracker)
if block:
try:
return ar.get()
except KeyboardInterrupt:
pass
return ar
@spin_after
@save_ids
def map(self, f, *sequences, **kwargs):
"""view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult
Parallel version of builtin `map`, load-balanced by this View.
`block`, and `chunksize` can be specified by keyword only.
Each `chunksize` elements will be a separate task, and will be
load-balanced. This lets individual elements be available for iteration
as soon as they arrive.
Parameters
----------
f : callable
function to be mapped
*sequences: one or more sequences of matching length
the sequences to be distributed and passed to `f`
block : bool [default self.block]
whether to wait for the result or not
track : bool
whether to create a MessageTracker to allow the user to
safely edit after arrays and buffers during non-copying
sends.
chunksize : int [default 1]
how many elements should be in each task.
ordered : bool [default True]
Whether the results should be gathered as they arrive, or enforce
the order of submission.
Only applies when iterating through AsyncMapResult as results arrive.
Has no effect when block=True.
Returns
-------
if block=False:
AsyncMapResult
An object like AsyncResult, but which reassembles the sequence of results
into a single list. AsyncMapResults can be iterated through before all
results are complete.
else:
the result of map(f,*sequences)
"""
# default
block = kwargs.get('block', self.block)
chunksize = kwargs.get('chunksize', 1)
ordered = kwargs.get('ordered', True)
keyset = set(kwargs.keys())
extra_keys = keyset.difference_update(set(['block', 'chunksize']))
if extra_keys:
raise TypeError("Invalid kwargs: %s" % list(extra_keys))
assert len(sequences) > 0, "must have some sequences to map onto!"
pf = ParallelFunction(self,
f,
block=block,
chunksize=chunksize,
ordered=ordered)
return pf.map(*sequences)
class LocalProcessLauncher(BaseLauncher):
"""Start and stop an external process in an asynchronous manner.
This will launch the external process with a working directory of
``self.work_dir``.
"""
# This is used to to construct self.args, which is passed to
# spawnProcess.
cmd_and_args = List([])
poll_frequency = Integer(100) # in ms
def __init__(self, work_dir=u'.', config=None, **kwargs):
super(LocalProcessLauncher, self).__init__(work_dir=work_dir,
config=config,
**kwargs)
self.process = None
self.poller = None
def find_args(self):
return self.cmd_and_args
def start(self):
self.log.debug("Starting %s: %r", self.__class__.__name__, self.args)
if self.state == 'before':
self.process = Popen(self.args,
stdout=PIPE,
stderr=PIPE,
stdin=PIPE,
env=os.environ,
cwd=self.work_dir)
if WINDOWS:
self.stdout = forward_read_events(self.process.stdout)
self.stderr = forward_read_events(self.process.stderr)
else:
self.stdout = self.process.stdout.fileno()
self.stderr = self.process.stderr.fileno()
self.loop.add_handler(self.stdout, self.handle_stdout,
self.loop.READ)
self.loop.add_handler(self.stderr, self.handle_stderr,
self.loop.READ)
self.poller = ioloop.PeriodicCallback(self.poll,
self.poll_frequency,
self.loop)
self.poller.start()
self.notify_start(self.process.pid)
else:
s = 'The process was already started and has state: %r' % self.state
raise ProcessStateError(s)
def stop(self):
return self.interrupt_then_kill()
def signal(self, sig):
if self.state == 'running':
if WINDOWS and sig != SIGINT:
# use Windows tree-kill for better child cleanup
check_output(
['taskkill', '-pid',
str(self.process.pid), '-t', '-f'])
else:
self.process.send_signal(sig)
def interrupt_then_kill(self, delay=2.0):
"""Send INT, wait a delay and then send KILL."""
try:
self.signal(SIGINT)
except Exception:
self.log.debug("interrupt failed")
pass
self.killer = ioloop.DelayedCallback(lambda: self.signal(SIGKILL),
delay * 1000, self.loop)
self.killer.start()
# callbacks, etc:
def handle_stdout(self, fd, events):
if WINDOWS:
line = self.stdout.recv()
else:
line = self.process.stdout.readline()
# a stopped process will be readable but return empty strings
if line:
self.log.debug(line[:-1])
else:
self.poll()
def handle_stderr(self, fd, events):
if WINDOWS:
line = self.stderr.recv()
else:
line = self.process.stderr.readline()
# a stopped process will be readable but return empty strings
if line:
self.log.debug(line[:-1])
else:
self.poll()
def poll(self):
status = self.process.poll()
if status is not None:
self.poller.stop()
self.loop.remove_handler(self.stdout)
self.loop.remove_handler(self.stderr)
self.notify_stop(dict(exit_code=status, pid=self.process.pid))
return status
class IPythonConsoleApp(ConnectionFileMixin):
name = 'ipython-console-mixin'
description = """
The IPython Mixin Console.
This class contains the common portions of console client (QtConsole,
ZMQ-based terminal console, etc). It is not a full console, in that
launched terminal subprocesses will not be able to accept input.
The Console using this mixing supports various extra features beyond
the single-process Terminal IPython shell, such as connecting to
existing kernel, via:
ipython <appname> --existing
as well as tunnel via SSH
"""
classes = classes
flags = Dict(flags)
aliases = Dict(aliases)
kernel_manager_class = KernelManager
kernel_client_class = BlockingKernelClient
kernel_argv = List(Unicode)
# frontend flags&aliases to be stripped when building kernel_argv
frontend_flags = Any(app_flags)
frontend_aliases = Any(app_aliases)
# create requested profiles by default, if they don't exist:
auto_create = CBool(True)
# connection info:
sshserver = Unicode(
'',
config=True,
help="""The SSH server to use to connect to the kernel.""")
sshkey = Unicode(
'',
config=True,
help="""Path to the ssh key to use for logging in to the ssh server."""
)
def _connection_file_default(self):
return 'kernel-%i.json' % os.getpid()
existing = CUnicode('',
config=True,
help="""Connect to an already running kernel""")
kernel_name = Unicode('python',
config=True,
help="""The name of the default kernel to start.""")
confirm_exit = CBool(
True,
config=True,
help="""
Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
to force a direct exit without any confirmation.""",
)
def build_kernel_argv(self, argv=None):
"""build argv to be passed to kernel subprocess
Override in subclasses if any args should be passed to the kernel
"""
self.kernel_argv = self.extra_args
def init_connection_file(self):
"""find the connection file, and load the info if found.
The current working directory and the current profile's security
directory will be searched for the file if it is not given by
absolute path.
When attempting to connect to an existing kernel and the `--existing`
argument does not match an existing file, it will be interpreted as a
fileglob, and the matching file in the current profile's security dir
with the latest access time will be used.
After this method is called, self.connection_file contains the *full path*
to the connection file, never just its name.
"""
if self.existing:
try:
cf = find_connection_file(self.existing)
except Exception:
self.log.critical(
"Could not find existing kernel connection file %s",
self.existing)
self.exit(1)
self.log.debug("Connecting to existing kernel: %s" % cf)
self.connection_file = cf
else:
# not existing, check if we are going to write the file
# and ensure that self.connection_file is a full path, not just the shortname
try:
cf = find_connection_file(self.connection_file)
except Exception:
# file might not exist
if self.connection_file == os.path.basename(
self.connection_file):
# just shortname, put it in security dir
cf = os.path.join(self.profile_dir.security_dir,
self.connection_file)
else:
cf = self.connection_file
self.connection_file = cf
try:
self.connection_file = filefind(
self.connection_file, ['.', self.profile_dir.security_dir])
except IOError:
self.log.debug("Connection File not found: %s",
self.connection_file)
return
# should load_connection_file only be used for existing?
# as it is now, this allows reusing ports if an existing
# file is requested
try:
self.load_connection_file()
except Exception:
self.log.error("Failed to load connection file: %r",
self.connection_file,
exc_info=True)
self.exit(1)
def init_ssh(self):
"""set up ssh tunnels, if needed."""
if not self.existing or (not self.sshserver and not self.sshkey):
return
self.load_connection_file()
transport = self.transport
ip = self.ip
if transport != 'tcp':
self.log.error("Can only use ssh tunnels with TCP sockets, not %s",
transport)
sys.exit(-1)
if self.sshkey and not self.sshserver:
# specifying just the key implies that we are connecting directly
self.sshserver = ip
ip = localhost()
# build connection dict for tunnels:
info = dict(ip=ip,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port)
self.log.info("Forwarding connections to %s via %s" %
(ip, self.sshserver))
# tunnels return a new set of ports, which will be on localhost:
self.ip = localhost()
try:
newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
except:
# even catch KeyboardInterrupt
self.log.error("Could not setup tunnels", exc_info=True)
self.exit(1)
self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
cf = self.connection_file
base, ext = os.path.splitext(cf)
base = os.path.basename(base)
self.connection_file = os.path.basename(base) + '-ssh' + ext
self.log.info("To connect another client via this tunnel, use:")
self.log.info("--existing %s" % self.connection_file)
def _new_connection_file(self):
cf = ''
while not cf:
# we don't need a 128b id to distinguish kernels, use more readable
# 48b node segment (12 hex chars). Users running more than 32k simultaneous
# kernels can subclass.
ident = str(uuid.uuid4()).split('-')[-1]
cf = os.path.join(self.profile_dir.security_dir,
'kernel-%s.json' % ident)
# only keep if it's actually new. Protect against unlikely collision
# in 48b random search space
cf = cf if not os.path.exists(cf) else ''
return cf
def init_kernel_manager(self):
# Don't let Qt or ZMQ swallow KeyboardInterupts.
if self.existing:
self.kernel_manager = None
return
signal.signal(signal.SIGINT, signal.SIG_DFL)
# Create a KernelManager and start a kernel.
try:
self.kernel_manager = self.kernel_manager_class(
ip=self.ip,
session=self.session,
transport=self.transport,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
connection_file=self.connection_file,
kernel_name=self.kernel_name,
parent=self,
ipython_dir=self.ipython_dir,
)
except NoSuchKernel:
self.log.critical("Could not find kernel %s", self.kernel_name)
self.exit(1)
self.kernel_manager.client_factory = self.kernel_client_class
# FIXME: remove special treatment of IPython kernels
kwargs = {}
if self.kernel_manager.ipython_kernel:
kwargs['extra_arguments'] = self.kernel_argv
self.kernel_manager.start_kernel(**kwargs)
atexit.register(self.kernel_manager.cleanup_ipc_files)
if self.sshserver:
# ssh, write new connection file
self.kernel_manager.write_connection_file()
# in case KM defaults / ssh writing changes things:
km = self.kernel_manager
self.shell_port = km.shell_port
self.iopub_port = km.iopub_port
self.stdin_port = km.stdin_port
self.hb_port = km.hb_port
self.connection_file = km.connection_file
atexit.register(self.kernel_manager.cleanup_connection_file)
def init_kernel_client(self):
if self.kernel_manager is not None:
self.kernel_client = self.kernel_manager.client()
else:
self.kernel_client = self.kernel_client_class(
session=self.session,
ip=self.ip,
transport=self.transport,
shell_port=self.shell_port,
iopub_port=self.iopub_port,
stdin_port=self.stdin_port,
hb_port=self.hb_port,
connection_file=self.connection_file,
parent=self,
)
self.kernel_client.start_channels()
def initialize(self, argv=None):
"""
Classes which mix this class in should call:
IPythonConsoleApp.initialize(self,argv)
"""
self.init_connection_file()
self.init_ssh()
self.init_kernel_manager()
self.init_kernel_client()
class MappingKernelManager(MultiKernelManager):
"""A KernelManager that handles notebook mapping and HTTP error handling"""
def _kernel_manager_class_default(self):
return "IPython.kernel.ioloop.IOLoopKernelManager"
kernel_argv = List(Unicode)
root_dir = Unicode(config=True)
def _root_dir_default(self):
try:
return self.parent.notebook_dir
except AttributeError:
return getcwd()
def _root_dir_changed(self, name, old, new):
"""Do a bit of validation of the root dir."""
if not os.path.isabs(new):
# If we receive a non-absolute path, make it absolute.
self.root_dir = os.path.abspath(new)
return
if not os.path.exists(new) or not os.path.isdir(new):
raise TraitError("kernel root dir %r is not a directory" % new)
#-------------------------------------------------------------------------
# Methods for managing kernels and sessions
#-------------------------------------------------------------------------
def _handle_kernel_died(self, kernel_id):
"""notice that a kernel died"""
self.log.warn("Kernel %s died, removing from map.", kernel_id)
self.remove_kernel(kernel_id)
def cwd_for_path(self, path):
"""Turn API path into absolute OS path."""
os_path = to_os_path(path, self.root_dir)
# in the case of notebooks and kernels not being on the same filesystem,
# walk up to root_dir if the paths don't exist
while not os.path.isdir(os_path) and os_path != self.root_dir:
os_path = os.path.dirname(os_path)
return os_path
def start_kernel(self,
kernel_id=None,
path=None,
kernel_name='python',
**kwargs):
"""Start a kernel for a session and return its kernel_id.
Parameters
----------
kernel_id : uuid
The uuid to associate the new kernel with. If this
is not None, this kernel will be persistent whenever it is
requested.
path : API path
The API path (unicode, '/' delimited) for the cwd.
Will be transformed to an OS path relative to root_dir.
kernel_name : str
The name identifying which kernel spec to launch. This is ignored if
an existing kernel is returned, but it may be checked in the future.
"""
if kernel_id is None:
if path is not None:
kwargs['cwd'] = self.cwd_for_path(path)
kernel_id = super(MappingKernelManager,
self).start_kernel(kernel_name=kernel_name,
**kwargs)
self.log.info("Kernel started: %s" % kernel_id)
self.log.debug("Kernel args: %r" % kwargs)
# register callback for failed auto-restart
self.add_restart_callback(
kernel_id,
lambda: self._handle_kernel_died(kernel_id),
'dead',
)
else:
self._check_kernel_id(kernel_id)
self.log.info("Using existing kernel: %s" % kernel_id)
return kernel_id
def shutdown_kernel(self, kernel_id, now=False):
"""Shutdown a kernel by kernel_id"""
self._check_kernel_id(kernel_id)
super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now)
def kernel_model(self, kernel_id):
"""Return a dictionary of kernel information described in the
JSON standard model."""
self._check_kernel_id(kernel_id)
model = {"id": kernel_id, "name": self._kernels[kernel_id].kernel_name}
return model
def list_kernels(self):
"""Returns a list of kernel_id's of kernels running."""
kernels = []
kernel_ids = super(MappingKernelManager, self).list_kernel_ids()
for kernel_id in kernel_ids:
model = self.kernel_model(kernel_id)
kernels.append(model)
return kernels
# override _check_kernel_id to raise 404 instead of KeyError
def _check_kernel_id(self, kernel_id):
"""Check a that a kernel_id exists and raise 404 if not."""
if kernel_id not in self:
raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
class InProcessKernel(Kernel):
#-------------------------------------------------------------------------
# InProcessKernel interface
#-------------------------------------------------------------------------
# The frontends connected to this kernel.
frontends = List(
Instance('IPython.kernel.inprocess.client.InProcessKernelClient'))
# The GUI environment that the kernel is running under. This need not be
# specified for the normal operation for the kernel, but is required for
# IPython's GUI support (including pylab). The default is 'inline' because
# it is safe under all GUI toolkits.
gui = Enum(('tk', 'gtk', 'wx', 'qt', 'qt4', 'inline'),
default_value='inline')
raw_input_str = Any()
stdout = Any()
stderr = Any()
#-------------------------------------------------------------------------
# Kernel interface
#-------------------------------------------------------------------------
shell_class = Type()
shell_streams = List()
control_stream = Any()
iopub_socket = Instance(DummySocket, ())
stdin_socket = Instance(DummySocket, ())
def __init__(self, **traits):
# When an InteractiveShell is instantiated by our base class, it binds
# the current values of sys.stdout and sys.stderr.
with self._redirected_io():
super(InProcessKernel, self).__init__(**traits)
self.iopub_socket.on_trait_change(self._io_dispatch, 'message_sent')
self.shell.kernel = self
def execute_request(self, stream, ident, parent):
""" Override for temporary IO redirection. """
with self._redirected_io():
super(InProcessKernel, self).execute_request(stream, ident, parent)
def start(self):
""" Override registration of dispatchers for streams. """
self.shell.exit_now = False
def _abort_queue(self, stream):
""" The in-process kernel doesn't abort requests. """
pass
def _raw_input(self, prompt, ident, parent):
# Flush output before making the request.
self.raw_input_str = None
sys.stderr.flush()
sys.stdout.flush()
# Send the input request.
content = json_clean(dict(prompt=prompt))
msg = self.session.msg('input_request', content, parent)
for frontend in self.frontends:
if frontend.session.session == parent['header']['session']:
frontend.stdin_channel.call_handlers(msg)
break
else:
logging.error('No frontend found for raw_input request')
return str()
# Await a response.
while self.raw_input_str is None:
frontend.stdin_channel.process_events()
return self.raw_input_str
#-------------------------------------------------------------------------
# Protected interface
#-------------------------------------------------------------------------
@contextmanager
def _redirected_io(self):
""" Temporarily redirect IO to the kernel.
"""
sys_stdout, sys_stderr = sys.stdout, sys.stderr
sys.stdout, sys.stderr = self.stdout, self.stderr
yield
sys.stdout, sys.stderr = sys_stdout, sys_stderr
#------ Trait change handlers --------------------------------------------
def _io_dispatch(self):
""" Called when a message is sent to the IO socket.
"""
ident, msg = self.session.recv(self.iopub_socket, copy=False)
for frontend in self.frontends:
frontend.iopub_channel.call_handlers(msg)
#------ Trait initializers -----------------------------------------------
def _log_default(self):
return logging.getLogger(__name__)
def _session_default(self):
from IPython.kernel.zmq.session import Session
return Session(parent=self)
def _shell_class_default(self):
return InProcessInteractiveShell
def _stdout_default(self):
from IPython.kernel.zmq.iostream import OutStream
return OutStream(self.session, self.iopub_socket, 'stdout', pipe=False)
def _stderr_default(self):
from IPython.kernel.zmq.iostream import OutStream
return OutStream(self.session, self.iopub_socket, 'stderr', pipe=False)
class TemplateExporter(Exporter):
"""
Exports notebooks into other file formats. Uses Jinja 2 templating engine
to output new formats. Inherit from this class if you are creating a new
template type along with new filters/preprocessors. If the filters/
preprocessors provided by default suffice, there is no need to inherit from
this class. Instead, override the template_file and file_extension
traits via a config file.
{filters}
"""
# finish the docstring
__doc__ = __doc__.format(filters='- ' +
'\n - '.join(default_filters.keys()))
template_file = Unicode(u'default',
config=True,
help="Name of the template file to use")
def _template_file_changed(self, name, old, new):
if new == 'default':
self.template_file = self.default_template
else:
self.template_file = new
self.template = None
self._load_template()
default_template = Unicode(u'')
template = Any()
environment = Any()
template_path = List(['.'], config=True)
def _template_path_changed(self, name, old, new):
self._load_template()
default_template_path = Unicode(
os.path.join("..", "templates"),
help="Path where the template files are located.")
template_skeleton_path = Unicode(
os.path.join("..", "templates", "skeleton"),
help="Path where the template skeleton files are located.")
#Jinja block definitions
jinja_comment_block_start = Unicode("", config=True)
jinja_comment_block_end = Unicode("", config=True)
jinja_variable_block_start = Unicode("", config=True)
jinja_variable_block_end = Unicode("", config=True)
jinja_logic_block_start = Unicode("", config=True)
jinja_logic_block_end = Unicode("", config=True)
#Extension that the template files use.
template_extension = Unicode(".tpl", config=True)
filters = Dict(
config=True,
help="""Dictionary of filters, by name and namespace, to add to the Jinja
environment.""")
def __init__(self, config=None, extra_loaders=None, **kw):
"""
Public constructor
Parameters
----------
config : config
User configuration instance.
extra_loaders : list[of Jinja Loaders]
ordered list of Jinja loader to find templates. Will be tried in order
before the default FileSystem ones.
template : str (optional, kw arg)
Template to use when exporting.
"""
super(TemplateExporter, self).__init__(config=config, **kw)
#Init
self._init_template()
self._init_environment(extra_loaders=extra_loaders)
self._init_preprocessors()
self._init_filters()
def _load_template(self):
"""Load the Jinja template object from the template file
This is a no-op if the template attribute is already defined,
or the Jinja environment is not setup yet.
This is triggered by various trait changes that would change the template.
"""
if self.template is not None:
return
# called too early, do nothing
if self.environment is None:
return
# Try different template names during conversion. First try to load the
# template by name with extension added, then try loading the template
# as if the name is explicitly specified, then try the name as a
# 'flavor', and lastly just try to load the template by module name.
module_name = self.__module__.rsplit('.', 1)[-1]
try_names = []
if self.template_file:
try_names.extend([
self.template_file + self.template_extension,
self.template_file,
module_name + '_' + self.template_file +
self.template_extension,
])
try_names.append(module_name + self.template_extension)
for try_name in try_names:
self.log.debug("Attempting to load template %s", try_name)
try:
self.template = self.environment.get_template(try_name)
except (TemplateNotFound, IOError):
pass
except Exception as e:
self.log.warn("Unexpected exception loading template: %s",
try_name,
exc_info=True)
else:
self.log.info("Loaded template %s", try_name)
break
def from_notebook_node(self, nb, resources=None, **kw):
"""
Convert a notebook from a notebook node instance.
Parameters
----------
nb : Notebook node
resources : dict (**kw)
of additional resources that can be accessed read/write by
preprocessors and filters.
"""
nb_copy, resources = super(TemplateExporter, self).from_notebook_node(
nb, resources, **kw)
self._load_template()
if self.template is not None:
output = self.template.render(nb=nb_copy, resources=resources)
else:
raise IOError('template file "%s" could not be found' %
self.template_file)
return output, resources
def register_filter(self, name, jinja_filter):
"""
Register a filter.
A filter is a function that accepts and acts on one string.
The filters are accesible within the Jinja templating engine.
Parameters
----------
name : str
name to give the filter in the Jinja engine
filter : filter
"""
if jinja_filter is None:
raise TypeError('filter')
isclass = isinstance(jinja_filter, type)
constructed = not isclass
#Handle filter's registration based on it's type
if constructed and isinstance(jinja_filter, py3compat.string_types):
#filter is a string, import the namespace and recursively call
#this register_filter method
filter_cls = import_item(jinja_filter)
return self.register_filter(name, filter_cls)
if constructed and hasattr(jinja_filter, '__call__'):
#filter is a function, no need to construct it.
self.environment.filters[name] = jinja_filter
return jinja_filter
elif isclass and isinstance(jinja_filter, MetaHasTraits):
#filter is configurable. Make sure to pass in new default for
#the enabled flag if one was specified.
filter_instance = jinja_filter(parent=self)
self.register_filter(name, filter_instance)
elif isclass:
#filter is not configurable, construct it
filter_instance = jinja_filter()
self.register_filter(name, filter_instance)
else:
#filter is an instance of something without a __call__
#attribute.
raise TypeError('filter')
def _init_template(self):
"""
Make sure a template name is specified. If one isn't specified, try to
build one from the information we know.
"""
self._template_file_changed('template_file', self.template_file,
self.template_file)
def _init_environment(self, extra_loaders=None):
"""
Create the Jinja templating environment.
"""
here = os.path.dirname(os.path.realpath(__file__))
loaders = []
if extra_loaders:
loaders.extend(extra_loaders)
paths = self.template_path
paths.extend([
os.path.join(here, self.default_template_path),
os.path.join(here, self.template_skeleton_path)
])
loaders.append(FileSystemLoader(paths))
self.environment = Environment(loader=ChoiceLoader(loaders),
extensions=JINJA_EXTENSIONS)
#Set special Jinja2 syntax that will not conflict with latex.
if self.jinja_logic_block_start:
self.environment.block_start_string = self.jinja_logic_block_start
if self.jinja_logic_block_end:
self.environment.block_end_string = self.jinja_logic_block_end
if self.jinja_variable_block_start:
self.environment.variable_start_string = self.jinja_variable_block_start
if self.jinja_variable_block_end:
self.environment.variable_end_string = self.jinja_variable_block_end
if self.jinja_comment_block_start:
self.environment.comment_start_string = self.jinja_comment_block_start
if self.jinja_comment_block_end:
self.environment.comment_end_string = self.jinja_comment_block_end
def _init_filters(self):
"""
Register all of the filters required for the exporter.
"""
#Add default filters to the Jinja2 environment
for key, value in default_filters.items():
self.register_filter(key, value)
#Load user filters. Overwrite existing filters if need be.
if self.filters:
for key, user_filter in self.filters.items():
self.register_filter(key, user_filter)
class AutoHandler(PrefilterHandler):
handler_name = Unicode('auto')
esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2])
def handle(self, line_info):
"""Handle lines which can be auto-executed, quoting if requested."""
line = line_info.line
ifun = line_info.ifun
the_rest = line_info.the_rest
pre = line_info.pre
esc = line_info.esc
continue_prompt = line_info.continue_prompt
obj = line_info.ofind(self.shell)['obj']
#print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest) # dbg
# This should only be active for single-line input!
if continue_prompt:
return line
force_auto = isinstance(obj, IPyAutocall)
# User objects sometimes raise exceptions on attribute access other
# than AttributeError (we've seen it in the past), so it's safest to be
# ultra-conservative here and catch all.
try:
auto_rewrite = obj.rewrite
except Exception:
auto_rewrite = True
if esc == ESC_QUOTE:
# Auto-quote splitting on whitespace
newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) )
elif esc == ESC_QUOTE2:
# Auto-quote whole string
newcmd = '%s("%s")' % (ifun,the_rest)
elif esc == ESC_PAREN:
newcmd = '%s(%s)' % (ifun,",".join(the_rest.split()))
else:
# Auto-paren.
if force_auto:
# Don't rewrite if it is already a call.
do_rewrite = not the_rest.startswith('(')
else:
if not the_rest:
# We only apply it to argument-less calls if the autocall
# parameter is set to 2.
do_rewrite = (self.shell.autocall >= 2)
elif the_rest.startswith('[') and hasattr(obj, '__getitem__'):
# Don't autocall in this case: item access for an object
# which is BOTH callable and implements __getitem__.
do_rewrite = False
else:
do_rewrite = True
# Figure out the rewritten command
if do_rewrite:
if the_rest.endswith(';'):
newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1])
else:
newcmd = '%s(%s)' % (ifun.rstrip(), the_rest)
else:
normal_handler = self.prefilter_manager.get_handler_by_name('normal')
return normal_handler.handle(line_info)
# Display the rewritten call
if auto_rewrite:
self.shell.auto_rewrite_input(newcmd)
return newcmd
class Kernel(Configurable):
#---------------------------------------------------------------------------
# Kernel interface
#---------------------------------------------------------------------------
# attribute to override with a GUI
eventloop = Any(None)
def _eventloop_changed(self, name, old, new):
"""schedule call to eventloop from IOLoop"""
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time()+0.1, self.enter_eventloop)
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
shell_class = Type(ZMQInteractiveShell)
session = Instance(Session)
profile_dir = Instance('IPython.core.profiledir.ProfileDir')
shell_streams = List()
control_stream = Instance(ZMQStream)
iopub_socket = Instance(zmq.Socket)
stdin_socket = Instance(zmq.Socket)
log = Instance(logging.Logger)
user_module = Any()
def _user_module_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_module = new
user_ns = Dict(default_value=None)
def _user_ns_changed(self, name, old, new):
if self.shell is not None:
self.shell.user_ns = new
self.shell.init_user_ns()
# identities:
int_id = Integer(-1)
ident = Unicode()
def _ident_default(self):
return unicode(uuid.uuid4())
# Private interface
# Time to sleep after flushing the stdout/err buffers in each execute
# cycle. While this introduces a hard limit on the minimal latency of the
# execute cycle, it helps prevent output synchronization problems for
# clients.
# Units are in seconds. The minimum zmq latency on local host is probably
# ~150 microseconds, set this to 500us for now. We may need to increase it
# a little if it's not enough after more interactive testing.
_execute_sleep = Float(0.0005, config=True)
# Frequency of the kernel's event loop.
# Units are in seconds, kernel subclasses for GUI toolkits may need to
# adapt to milliseconds.
_poll_interval = Float(0.05, config=True)
# If the shutdown was requested over the network, we leave here the
# necessary reply message so it can be sent by our registered atexit
# handler. This ensures that the reply is only sent to clients truly at
# the end of our shutdown process (which happens after the underlying
# IPython shell's own shutdown).
_shutdown_message = None
# This is a dict of port number that the kernel is listening on. It is set
# by record_ports and used by connect_request.
_recorded_ports = Dict()
# A reference to the Python builtin 'raw_input' function.
# (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
_sys_raw_input = Any()
# set of aborted msg_ids
aborted = Set()
def __init__(self, **kwargs):
super(Kernel, self).__init__(**kwargs)
# Initialize the InteractiveShell subclass
self.shell = self.shell_class.instance(config=self.config,
profile_dir = self.profile_dir,
user_module = self.user_module,
user_ns = self.user_ns,
)
self.shell.displayhook.session = self.session
self.shell.displayhook.pub_socket = self.iopub_socket
self.shell.displayhook.topic = self._topic('pyout')
self.shell.display_pub.session = self.session
self.shell.display_pub.pub_socket = self.iopub_socket
self.shell.data_pub.session = self.session
self.shell.data_pub.pub_socket = self.iopub_socket
# TMP - hack while developing
self.shell._reply_content = None
# Build dict of handlers for message types
msg_types = [ 'execute_request', 'complete_request',
'object_info_request', 'history_request',
'kernel_info_request',
'connect_request', 'shutdown_request',
'apply_request',
]
self.shell_handlers = {}
for msg_type in msg_types:
self.shell_handlers[msg_type] = getattr(self, msg_type)
control_msg_types = msg_types + [ 'clear_request', 'abort_request' ]
self.control_handlers = {}
for msg_type in control_msg_types:
self.control_handlers[msg_type] = getattr(self, msg_type)
def dispatch_control(self, msg):
"""dispatch control requests"""
idents,msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Control Message", exc_info=True)
return
self.log.debug("Control received: %s", msg)
header = msg['header']
msg_id = header['msg_id']
msg_type = header['msg_type']
handler = self.control_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
else:
try:
handler(self.control_stream, idents, msg)
except Exception:
self.log.error("Exception in control handler:", exc_info=True)
def dispatch_shell(self, stream, msg):
"""dispatch shell requests"""
# flush control requests first
if self.control_stream:
self.control_stream.flush()
idents,msg = self.session.feed_identities(msg, copy=False)
try:
msg = self.session.unserialize(msg, content=True, copy=False)
except:
self.log.error("Invalid Message", exc_info=True)
return
header = msg['header']
msg_id = header['msg_id']
msg_type = msg['header']['msg_type']
# Print some info about this message and leave a '--->' marker, so it's
# easier to trace visually the message chain when debugging. Each
# handler prints its message at the end.
self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
self.log.debug(' Content: %s\n --->\n ', msg['content'])
if msg_id in self.aborted:
self.aborted.remove(msg_id)
# is it safe to assume a msg_id will not be resubmitted?
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status' : 'aborted'}
md = {'engine' : self.ident}
md.update(status)
reply_msg = self.session.send(stream, reply_type, metadata=md,
content=status, parent=msg, ident=idents)
return
handler = self.shell_handlers.get(msg_type, None)
if handler is None:
self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
else:
# ensure default_int_handler during handler call
sig = signal(SIGINT, default_int_handler)
try:
handler(stream, idents, msg)
except Exception:
self.log.error("Exception in message handler:", exc_info=True)
finally:
signal(SIGINT, sig)
def enter_eventloop(self):
"""enter eventloop"""
self.log.info("entering eventloop")
# restore default_int_handler
signal(SIGINT, default_int_handler)
while self.eventloop is not None:
try:
self.eventloop(self)
except KeyboardInterrupt:
# Ctrl-C shouldn't crash the kernel
self.log.error("KeyboardInterrupt caught in kernel")
continue
else:
# eventloop exited cleanly, this means we should stop (right?)
self.eventloop = None
break
self.log.info("exiting eventloop")
def start(self):
"""register dispatchers for streams"""
self.shell.exit_now = False
if self.control_stream:
self.control_stream.on_recv(self.dispatch_control, copy=False)
def make_dispatcher(stream):
def dispatcher(msg):
return self.dispatch_shell(stream, msg)
return dispatcher
for s in self.shell_streams:
s.on_recv(make_dispatcher(s), copy=False)
def do_one_iteration(self):
"""step eventloop just once"""
if self.control_stream:
self.control_stream.flush()
for stream in self.shell_streams:
# handle at most one request per iteration
stream.flush(zmq.POLLIN, 1)
stream.flush(zmq.POLLOUT)
def record_ports(self, ports):
"""Record the ports that this kernel is using.
The creator of the Kernel instance must call this methods if they
want the :meth:`connect_request` method to return the port numbers.
"""
self._recorded_ports = ports
#---------------------------------------------------------------------------
# Kernel request handlers
#---------------------------------------------------------------------------
def _make_metadata(self, other=None):
"""init metadata dict, for execute/apply_reply"""
new_md = {
'dependencies_met' : True,
'engine' : self.ident,
'started': datetime.now(),
}
if other:
new_md.update(other)
return new_md
def _publish_pyin(self, code, parent, execution_count):
"""Publish the code request on the pyin stream."""
self.session.send(self.iopub_socket, u'pyin',
{u'code':code, u'execution_count': execution_count},
parent=parent, ident=self._topic('pyin')
)
def _publish_status(self, status, parent=None):
"""send status (busy/idle) on IOPub"""
self.session.send(self.iopub_socket,
u'status',
{u'execution_state': status},
parent=parent,
ident=self._topic('status'),
)
def execute_request(self, stream, ident, parent):
"""handle an execute_request"""
self._publish_status(u'busy', parent)
try:
content = parent[u'content']
code = content[u'code']
silent = content[u'silent']
store_history = content.get(u'store_history', not silent)
except:
self.log.error("Got bad msg: ")
self.log.error("%s", parent)
return
md = self._make_metadata(parent['metadata'])
shell = self.shell # we'll need this a lot here
# Replace raw_input. Note that is not sufficient to replace
# raw_input in the user namespace.
if content.get('allow_stdin', False):
raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
else:
raw_input = lambda prompt='' : self._no_raw_input()
if py3compat.PY3:
self._sys_raw_input = __builtin__.input
__builtin__.input = raw_input
else:
self._sys_raw_input = __builtin__.raw_input
__builtin__.raw_input = raw_input
# Set the parent message of the display hook and out streams.
shell.displayhook.set_parent(parent)
shell.display_pub.set_parent(parent)
shell.data_pub.set_parent(parent)
try:
sys.stdout.set_parent(parent)
except AttributeError:
pass
try:
sys.stderr.set_parent(parent)
except AttributeError:
pass
# Re-broadcast our input for the benefit of listening clients, and
# start computing output
if not silent:
self._publish_pyin(code, parent, shell.execution_count)
reply_content = {}
try:
# FIXME: the shell calls the exception handler itself.
shell.run_cell(code, store_history=store_history, silent=silent)
except:
status = u'error'
# FIXME: this code right now isn't being used yet by default,
# because the run_cell() call above directly fires off exception
# reporting. This code, therefore, is only active in the scenario
# where runlines itself has an unhandled exception. We need to
# uniformize this, for all exception construction to come from a
# single location in the codbase.
etype, evalue, tb = sys.exc_info()
tb_list = traceback.format_exception(etype, evalue, tb)
reply_content.update(shell._showtraceback(etype, evalue, tb_list))
else:
status = u'ok'
finally:
# Restore raw_input.
if py3compat.PY3:
__builtin__.input = self._sys_raw_input
else:
__builtin__.raw_input = self._sys_raw_input
reply_content[u'status'] = status
# Return the execution counter so clients can display prompts
reply_content['execution_count'] = shell.execution_count - 1
# FIXME - fish exception info out of shell, possibly left there by
# runlines. We'll need to clean up this logic later.
if shell._reply_content is not None:
reply_content.update(shell._reply_content)
e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute')
reply_content['engine_info'] = e_info
# reset after use
shell._reply_content = None
if 'traceback' in reply_content:
self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback']))
# At this point, we can tell whether the main code execution succeeded
# or not. If it did, we proceed to evaluate user_variables/expressions
if reply_content['status'] == 'ok':
reply_content[u'user_variables'] = \
shell.user_variables(content.get(u'user_variables', []))
reply_content[u'user_expressions'] = \
shell.user_expressions(content.get(u'user_expressions', {}))
else:
# If there was an error, don't even try to compute variables or
# expressions
reply_content[u'user_variables'] = {}
reply_content[u'user_expressions'] = {}
# Payloads should be retrieved regardless of outcome, so we can both
# recover partial output (that could have been generated early in a
# block, before an error) and clear the payload system always.
reply_content[u'payload'] = shell.payload_manager.read_payload()
# Be agressive about clearing the payload because we don't want
# it to sit in memory until the next execute_request comes in.
shell.payload_manager.clear_payload()
# Flush output before sending the reply.
sys.stdout.flush()
sys.stderr.flush()
# FIXME: on rare occasions, the flush doesn't seem to make it to the
# clients... This seems to mitigate the problem, but we definitely need
# to better understand what's going on.
if self._execute_sleep:
time.sleep(self._execute_sleep)
# Send the reply.
reply_content = json_clean(reply_content)
md['status'] = reply_content['status']
if reply_content['status'] == 'error' and \
reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
reply_msg = self.session.send(stream, u'execute_reply',
reply_content, parent, metadata=md,
ident=ident)
self.log.debug("%s", reply_msg)
if not silent and reply_msg['content']['status'] == u'error':
self._abort_queues()
self._publish_status(u'idle', parent)
def complete_request(self, stream, ident, parent):
txt, matches = self._complete(parent)
matches = {'matches' : matches,
'matched_text' : txt,
'status' : 'ok'}
matches = json_clean(matches)
completion_msg = self.session.send(stream, 'complete_reply',
matches, parent, ident)
self.log.debug("%s", completion_msg)
def object_info_request(self, stream, ident, parent):
content = parent['content']
object_info = self.shell.object_inspect(content['oname'],
detail_level = content.get('detail_level', 0)
)
# Before we send this object over, we scrub it for JSON usage
oinfo = json_clean(object_info)
msg = self.session.send(stream, 'object_info_reply',
oinfo, parent, ident)
self.log.debug("%s", msg)
def history_request(self, stream, ident, parent):
# We need to pull these out, as passing **kwargs doesn't work with
# unicode keys before Python 2.6.5.
hist_access_type = parent['content']['hist_access_type']
raw = parent['content']['raw']
output = parent['content']['output']
if hist_access_type == 'tail':
n = parent['content']['n']
hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
include_latest=True)
elif hist_access_type == 'range':
session = parent['content']['session']
start = parent['content']['start']
stop = parent['content']['stop']
hist = self.shell.history_manager.get_range(session, start, stop,
raw=raw, output=output)
elif hist_access_type == 'search':
n = parent['content'].get('n')
unique = parent['content'].get('unique', False)
pattern = parent['content']['pattern']
hist = self.shell.history_manager.search(
pattern, raw=raw, output=output, n=n, unique=unique)
else:
hist = []
hist = list(hist)
content = {'history' : hist}
content = json_clean(content)
msg = self.session.send(stream, 'history_reply',
content, parent, ident)
self.log.debug("Sending history reply with %i entries", len(hist))
def connect_request(self, stream, ident, parent):
if self._recorded_ports is not None:
content = self._recorded_ports.copy()
else:
content = {}
msg = self.session.send(stream, 'connect_reply',
content, parent, ident)
self.log.debug("%s", msg)
def kernel_info_request(self, stream, ident, parent):
vinfo = {
'protocol_version': protocol_version,
'ipython_version': ipython_version,
'language_version': language_version,
'language': 'python',
}
msg = self.session.send(stream, 'kernel_info_reply',
vinfo, parent, ident)
self.log.debug("%s", msg)
def shutdown_request(self, stream, ident, parent):
self.shell.exit_now = True
content = dict(status='ok')
content.update(parent['content'])
self.session.send(stream, u'shutdown_reply', content, parent, ident=ident)
# same content, but different msg_id for broadcasting on IOPub
self._shutdown_message = self.session.msg(u'shutdown_reply',
content, parent
)
self._at_shutdown()
# call sys.exit after a short delay
loop = ioloop.IOLoop.instance()
loop.add_timeout(time.time()+0.1, loop.stop)
#---------------------------------------------------------------------------
# Engine methods
#---------------------------------------------------------------------------
def apply_request(self, stream, ident, parent):
try:
content = parent[u'content']
bufs = parent[u'buffers']
msg_id = parent['header']['msg_id']
except:
self.log.error("Got bad msg: %s", parent, exc_info=True)
return
self._publish_status(u'busy', parent)
# Set the parent message of the display hook and out streams.
shell = self.shell
shell.displayhook.set_parent(parent)
shell.display_pub.set_parent(parent)
shell.data_pub.set_parent(parent)
try:
sys.stdout.set_parent(parent)
except AttributeError:
pass
try:
sys.stderr.set_parent(parent)
except AttributeError:
pass
# pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
# self.iopub_socket.send(pyin_msg)
# self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
md = self._make_metadata(parent['metadata'])
try:
working = shell.user_ns
prefix = "_"+str(msg_id).replace("-","")+"_"
f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
fname = getattr(f, '__name__', 'f')
fname = prefix+"f"
argname = prefix+"args"
kwargname = prefix+"kwargs"
resultname = prefix+"result"
ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
# print ns
working.update(ns)
code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname)
try:
exec code in shell.user_global_ns, shell.user_ns
result = working.get(resultname)
finally:
for key in ns.iterkeys():
working.pop(key)
result_buf = serialize_object(result,
buffer_threshold=self.session.buffer_threshold,
item_threshold=self.session.item_threshold,
)
except:
# invoke IPython traceback formatting
shell.showtraceback()
# FIXME - fish exception info out of shell, possibly left there by
# run_code. We'll need to clean up this logic later.
reply_content = {}
if shell._reply_content is not None:
reply_content.update(shell._reply_content)
e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply')
reply_content['engine_info'] = e_info
# reset after use
shell._reply_content = None
self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent,
ident=self._topic('pyerr'))
self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback']))
result_buf = []
if reply_content['ename'] == 'UnmetDependency':
md['dependencies_met'] = False
else:
reply_content = {'status' : 'ok'}
# put 'ok'/'error' status in header, for scheduler introspection:
md['status'] = reply_content['status']
# flush i/o
sys.stdout.flush()
sys.stderr.flush()
reply_msg = self.session.send(stream, u'apply_reply', reply_content,
parent=parent, ident=ident,buffers=result_buf, metadata=md)
self._publish_status(u'idle', parent)
#---------------------------------------------------------------------------
# Control messages
#---------------------------------------------------------------------------
def abort_request(self, stream, ident, parent):
"""abort a specifig msg by id"""
msg_ids = parent['content'].get('msg_ids', None)
if isinstance(msg_ids, basestring):
msg_ids = [msg_ids]
if not msg_ids:
self.abort_queues()
for mid in msg_ids:
self.aborted.add(str(mid))
content = dict(status='ok')
reply_msg = self.session.send(stream, 'abort_reply', content=content,
parent=parent, ident=ident)
self.log.debug("%s", reply_msg)
def clear_request(self, stream, idents, parent):
"""Clear our namespace."""
self.shell.reset(False)
msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent,
content = dict(status='ok'))
#---------------------------------------------------------------------------
# Protected interface
#---------------------------------------------------------------------------
def _wrap_exception(self, method=None):
# import here, because _wrap_exception is only used in parallel,
# and parallel has higher min pyzmq version
from IPython.parallel.error import wrap_exception
e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
content = wrap_exception(e_info)
return content
def _topic(self, topic):
"""prefixed topic for IOPub messages"""
if self.int_id >= 0:
base = "engine.%i" % self.int_id
else:
base = "kernel.%s" % self.ident
return py3compat.cast_bytes("%s.%s" % (base, topic))
def _abort_queues(self):
for stream in self.shell_streams:
if stream:
self._abort_queue(stream)
def _abort_queue(self, stream):
poller = zmq.Poller()
poller.register(stream.socket, zmq.POLLIN)
while True:
idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
if msg is None:
return
self.log.info("Aborting:")
self.log.info("%s", msg)
msg_type = msg['header']['msg_type']
reply_type = msg_type.split('_')[0] + '_reply'
status = {'status' : 'aborted'}
md = {'engine' : self.ident}
md.update(status)
reply_msg = self.session.send(stream, reply_type, metadata=md,
content=status, parent=msg, ident=idents)
self.log.debug("%s", reply_msg)
# We need to wait a bit for requests to come in. This can probably
# be set shorter for true asynchronous clients.
poller.poll(50)
def _no_raw_input(self):
"""Raise StdinNotImplentedError if active frontend doesn't support
stdin."""
raise StdinNotImplementedError("raw_input was called, but this "
"frontend does not support stdin.")
def _raw_input(self, prompt, ident, parent):
# Flush output before making the request.
sys.stderr.flush()
sys.stdout.flush()
# Send the input request.
content = json_clean(dict(prompt=prompt))
self.session.send(self.stdin_socket, u'input_request', content, parent,
ident=ident)
# Await a response.
while True:
try:
ident, reply = self.session.recv(self.stdin_socket, 0)
except Exception:
self.log.warn("Invalid Message:", exc_info=True)
else:
break
try:
value = reply['content']['value']
except:
self.log.error("Got bad raw_input reply: ")
self.log.error("%s", parent)
value = ''
if value == '\x04':
# EOF
raise EOFError
return value
def _complete(self, msg):
c = msg['content']
try:
cpos = int(c['cursor_pos'])
except:
# If we don't get something that we can convert to an integer, at
# least attempt the completion guessing the cursor is at the end of
# the text, if there's any, and otherwise of the line
cpos = len(c['text'])
if cpos==0:
cpos = len(c['line'])
return self.shell.complete(c['text'], c['line'], cpos)
def _at_shutdown(self):
"""Actions taken at shutdown by the kernel, called by python's atexit.
"""
# io.rprint("Kernel at_shutdown") # dbg
if self._shutdown_message is not None:
self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))
self.log.debug("%s", self._shutdown_message)
[ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
class IPControllerApp(BaseParallelApplication):
name = u'ipcontroller'
description = _description
examples = _examples
classes = [
ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB
] + real_dbs
# change default to True
auto_create = Bool(
True,
config=True,
help="""Whether to create profile dir if it doesn't exist.""")
reuse_files = Bool(False,
config=True,
help="""Whether to reuse existing json connection files.
If False, connection files will be removed on a clean exit.
""")
restore_engines = Bool(False,
config=True,
help="""Reload engine state from JSON file
""")
ssh_server = Unicode(
u'',
config=True,
help="""ssh url for clients to use when connecting to the Controller
processes. It should be of the form: [user@]server[:port]. The
Controller's listening addresses must be accessible from the ssh server""",
)
engine_ssh_server = Unicode(
u'',
config=True,
help="""ssh url for engines to use when connecting to the Controller
processes. It should be of the form: [user@]server[:port]. The
Controller's listening addresses must be accessible from the ssh server""",
)
location = Unicode(
u'',
config=True,
help=
"""The external IP or domain name of the Controller, used for disambiguating
engine and client connections.""",
)
import_statements = List(
[],
config=True,
help=
"import statements to be run at startup. Necessary in some environments"
)
use_threads = Bool(
False,
config=True,
help='Use threads instead of processes for the schedulers',
)
engine_json_file = Unicode(
'ipcontroller-engine.json',
config=True,
help="JSON filename where engine connection info will be stored.")
client_json_file = Unicode(
'ipcontroller-client.json',
config=True,
help="JSON filename where client connection info will be stored.")
def _cluster_id_changed(self, name, old, new):
super(IPControllerApp, self)._cluster_id_changed(name, old, new)
self.engine_json_file = "%s-engine.json" % self.name
self.client_json_file = "%s-client.json" % self.name
# internal
children = List()
mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
def _use_threads_changed(self, name, old, new):
self.mq_class = 'zmq.devices.%sMonitoredQueue' % ('Thread' if new else
'Process')
write_connection_files = Bool(
True,
help="""Whether to write connection files to disk.
True in all cases other than runs with `reuse_files=True` *after the first*
""")
aliases = Dict(aliases)
flags = Dict(flags)
def save_connection_dict(self, fname, cdict):
"""save a connection dict to json file."""
c = self.config
url = cdict['registration']
location = cdict['location']
if not location:
if public_ips():
location = public_ips()[-1]
else:
self.log.warn(
"Could not identify this machine's IP, assuming %s."
" You may need to specify '--location=<external_ip_address>' to help"
" IPython decide when to connect via loopback." %
localhost())
location = localhost()
cdict['location'] = location
fname = os.path.join(self.profile_dir.security_dir, fname)
self.log.info("writing connection info to %s", fname)
with open(fname, 'w') as f:
f.write(json.dumps(cdict, indent=2))
os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR)
def load_config_from_json(self):
"""load config from existing json connector files."""
c = self.config
self.log.debug("loading config from JSON")
# load engine config
fname = os.path.join(self.profile_dir.security_dir,
self.engine_json_file)
self.log.info("loading connection info from %s", fname)
with open(fname) as f:
ecfg = json.loads(f.read())
# json gives unicode, Session.key wants bytes
c.Session.key = ecfg['key'].encode('ascii')
xport, ip = ecfg['interface'].split('://')
c.HubFactory.engine_ip = ip
c.HubFactory.engine_transport = xport
self.location = ecfg['location']
if not self.engine_ssh_server:
self.engine_ssh_server = ecfg['ssh']
# load client config
fname = os.path.join(self.profile_dir.security_dir,
self.client_json_file)
self.log.info("loading connection info from %s", fname)
with open(fname) as f:
ccfg = json.loads(f.read())
for key in ('key', 'registration', 'pack', 'unpack',
'signature_scheme'):
assert ccfg[key] == ecfg[
key], "mismatch between engine and client info: %r" % key
xport, addr = ccfg['interface'].split('://')
c.HubFactory.client_transport = xport
c.HubFactory.client_ip = ip
if not self.ssh_server:
self.ssh_server = ccfg['ssh']
# load port config:
c.HubFactory.regport = ecfg['registration']
c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong'])
c.HubFactory.control = (ccfg['control'], ecfg['control'])
c.HubFactory.mux = (ccfg['mux'], ecfg['mux'])
c.HubFactory.task = (ccfg['task'], ecfg['task'])
c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub'])
c.HubFactory.notifier_port = ccfg['notification']
def cleanup_connection_files(self):
if self.reuse_files:
self.log.debug("leaving JSON connection files for reuse")
return
self.log.debug("cleaning up JSON connection files")
for f in (self.client_json_file, self.engine_json_file):
f = os.path.join(self.profile_dir.security_dir, f)
try:
os.remove(f)
except Exception as e:
self.log.error("Failed to cleanup connection file: %s", e)
else:
self.log.debug(u"removed %s", f)
def load_secondary_config(self):
"""secondary config, loading from JSON and setting defaults"""
if self.reuse_files:
try:
self.load_config_from_json()
except (AssertionError, IOError) as e:
self.log.error("Could not load config from JSON: %s" % e)
else:
# successfully loaded config from JSON, and reuse=True
# no need to wite back the same file
self.write_connection_files = False
self.log.debug("Config changed")
self.log.debug(repr(self.config))
def init_hub(self):
c = self.config
self.do_import_statements()
try:
self.factory = HubFactory(config=c, log=self.log)
# self.start_logging()
self.factory.init_hub()
except TraitError:
raise
except Exception:
self.log.error("Couldn't construct the Controller", exc_info=True)
self.exit(1)
if self.write_connection_files:
# save to new json config files
f = self.factory
base = {
'key': f.session.key.decode('ascii'),
'location': self.location,
'pack': f.session.packer,
'unpack': f.session.unpacker,
'signature_scheme': f.session.signature_scheme,
}
cdict = {'ssh': self.ssh_server}
cdict.update(f.client_info)
cdict.update(base)
self.save_connection_dict(self.client_json_file, cdict)
edict = {'ssh': self.engine_ssh_server}
edict.update(f.engine_info)
edict.update(base)
self.save_connection_dict(self.engine_json_file, edict)
fname = "engines%s.json" % self.cluster_id
self.factory.hub.engine_state_file = os.path.join(
self.profile_dir.log_dir, fname)
if self.restore_engines:
self.factory.hub._load_engine_state()
# load key into config so other sessions in this process (TaskScheduler)
# have the same value
self.config.Session.key = self.factory.session.key
def init_schedulers(self):
children = self.children
mq = import_item(str(self.mq_class))
f = self.factory
ident = f.session.bsession
# disambiguate url, in case of *
monitor_url = disambiguate_url(f.monitor_url)
# maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
# IOPub relay (in a Process)
q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A', b'iopub')
q.bind_in(f.client_url('iopub'))
q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub")
q.bind_out(f.engine_url('iopub'))
q.setsockopt_out(zmq.SUBSCRIBE, b'')
q.connect_mon(monitor_url)
q.daemon = True
children.append(q)
# Multiplexer Queue (in a Process)
q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
q.bind_in(f.client_url('mux'))
q.setsockopt_in(zmq.IDENTITY, b'mux_in')
q.bind_out(f.engine_url('mux'))
q.setsockopt_out(zmq.IDENTITY, b'mux_out')
q.connect_mon(monitor_url)
q.daemon = True
children.append(q)
# Control Queue (in a Process)
q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
q.bind_in(f.client_url('control'))
q.setsockopt_in(zmq.IDENTITY, b'control_in')
q.bind_out(f.engine_url('control'))
q.setsockopt_out(zmq.IDENTITY, b'control_out')
q.connect_mon(monitor_url)
q.daemon = True
children.append(q)
if 'TaskScheduler.scheme_name' in self.config:
scheme = self.config.TaskScheduler.scheme_name
else:
scheme = TaskScheduler.scheme_name.get_default_value()
# Task Queue (in a Process)
if scheme == 'pure':
self.log.warn("task::using pure DEALER Task scheduler")
q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
# q.setsockopt_out(zmq.HWM, hub.hwm)
q.bind_in(f.client_url('task'))
q.setsockopt_in(zmq.IDENTITY, b'task_in')
q.bind_out(f.engine_url('task'))
q.setsockopt_out(zmq.IDENTITY, b'task_out')
q.connect_mon(monitor_url)
q.daemon = True
children.append(q)
elif scheme == 'none':
self.log.warn("task::using no Task scheduler")
else:
self.log.info("task::using Python %s Task scheduler" % scheme)
sargs = (
f.client_url('task'),
f.engine_url('task'),
monitor_url,
disambiguate_url(f.client_url('notification')),
disambiguate_url(f.client_url('registration')),
)
kwargs = dict(logname='scheduler',
loglevel=self.log_level,
log_url=self.log_url,
config=dict(self.config))
if 'Process' in self.mq_class:
# run the Python scheduler in a Process
q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
q.daemon = True
children.append(q)
else:
# single-threaded Controller
kwargs['in_thread'] = True
launch_scheduler(*sargs, **kwargs)
# set unlimited HWM for all relay devices
if hasattr(zmq, 'SNDHWM'):
q = children[0]
q.setsockopt_in(zmq.RCVHWM, 0)
q.setsockopt_out(zmq.SNDHWM, 0)
for q in children[1:]:
if not hasattr(q, 'setsockopt_in'):
continue
q.setsockopt_in(zmq.SNDHWM, 0)
q.setsockopt_in(zmq.RCVHWM, 0)
q.setsockopt_out(zmq.SNDHWM, 0)
q.setsockopt_out(zmq.RCVHWM, 0)
q.setsockopt_mon(zmq.SNDHWM, 0)
def terminate_children(self):
child_procs = []
for child in self.children:
if isinstance(child, ProcessMonitoredQueue):
child_procs.append(child.launcher)
elif isinstance(child, Process):
child_procs.append(child)
if child_procs:
self.log.critical("terminating children...")
for child in child_procs:
try:
child.terminate()
except OSError:
# already dead
pass
def handle_signal(self, sig, frame):
self.log.critical("Received signal %i, shutting down", sig)
self.terminate_children()
self.loop.stop()
def init_signal(self):
for sig in (SIGINT, SIGABRT, SIGTERM):
signal(sig, self.handle_signal)
def do_import_statements(self):
statements = self.import_statements
for s in statements:
try:
self.log.msg("Executing statement: '%s'" % s)
exec(s, globals(), locals())
except:
self.log.msg("Error running statement: %s" % s)
def forward_logging(self):
if self.log_url:
self.log.info("Forwarding logging to %s" % self.log_url)
context = zmq.Context.instance()
lsock = context.socket(zmq.PUB)
lsock.connect(self.log_url)
handler = PUBHandler(lsock)
handler.root_topic = 'controller'
handler.setLevel(self.log_level)
self.log.addHandler(handler)
@catch_config_error
def initialize(self, argv=None):
super(IPControllerApp, self).initialize(argv)
self.forward_logging()
self.load_secondary_config()
self.init_hub()
self.init_schedulers()
def start(self):
# Start the subprocesses:
self.factory.start()
# children must be started before signals are setup,
# otherwise signal-handling will fire multiple times
for child in self.children:
child.start()
self.init_signal()
self.write_pid_file(overwrite=True)
try:
self.factory.loop.start()
except KeyboardInterrupt:
self.log.critical("Interrupted, Exiting...\n")
finally:
self.cleanup_connection_files()
class HistoryManager(HistoryAccessor):
"""A class to organize all history-related functionality in one place.
"""
# Public interface
# An instance of the IPython shell we are attached to
shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
# Lists to hold processed and raw history. These start with a blank entry
# so that we can index them starting from 1
input_hist_parsed = List([""])
input_hist_raw = List([""])
# A list of directories visited during session
dir_hist = List()
def _dir_hist_default(self):
try:
return [py3compat.getcwd()]
except OSError:
return []
# A dict of output history, keyed with ints from the shell's
# execution count.
output_hist = Dict()
# The text/plain repr of outputs.
output_hist_reprs = Dict()
# The number of the current session in the history database
session_number = Integer()
db_log_output = Bool(
False,
config=True,
help="Should the history database include output? (default: no)")
db_cache_size = Integer(
0,
config=True,
help=
"Write to database every x commands (higher values save disk access & power).\n"
"Values of 1 or less effectively disable caching.")
# The input and output caches
db_input_cache = List()
db_output_cache = List()
# History saving in separate thread
save_thread = Instance('IPython.core.history.HistorySavingThread')
try: # Event is a function returning an instance of _Event...
save_flag = Instance(threading._Event)
except AttributeError: # ...until Python 3.3, when it's a class.
save_flag = Instance(threading.Event)
# Private interface
# Variables used to store the three last inputs from the user. On each new
# history update, we populate the user's namespace with these, shifted as
# necessary.
_i00 = Unicode(u'')
_i = Unicode(u'')
_ii = Unicode(u'')
_iii = Unicode(u'')
# A regex matching all forms of the exit command, so that we don't store
# them in the history (it's annoying to rewind the first entry and land on
# an exit call).
_exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
def __init__(self, shell=None, config=None, **traits):
"""Create a new history manager associated with a shell instance.
"""
# We need a pointer back to the shell for various tasks.
super(HistoryManager, self).__init__(shell=shell,
config=config,
**traits)
self.save_flag = threading.Event()
self.db_input_cache_lock = threading.Lock()
self.db_output_cache_lock = threading.Lock()
if self.enabled and self.hist_file != ':memory:':
self.save_thread = HistorySavingThread(self)
self.save_thread.start()
self.new_session()
def _get_hist_file_name(self, profile=None):
"""Get default history file name based on the Shell's profile.
The profile parameter is ignored, but must exist for compatibility with
the parent class."""
profile_dir = self.shell.profile_dir.location
return os.path.join(profile_dir, 'history.sqlite')
@needs_sqlite
def new_session(self, conn=None):
"""Get a new session number."""
if conn is None:
conn = self.db
with conn:
cur = conn.execute(
"""INSERT INTO sessions VALUES (NULL, ?, NULL,
NULL, "") """, (datetime.datetime.now(), ))
self.session_number = cur.lastrowid
def end_session(self):
"""Close the database session, filling in the end time and line count."""
self.writeout_cache()
with self.db:
self.db.execute(
"""UPDATE sessions SET end=?, num_cmds=? WHERE
session==?""",
(datetime.datetime.now(), len(self.input_hist_parsed) - 1,
self.session_number))
self.session_number = 0
def name_session(self, name):
"""Give the current session a name in the history database."""
with self.db:
self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
(name, self.session_number))
def reset(self, new_session=True):
"""Clear the session history, releasing all object references, and
optionally open a new session."""
self.output_hist.clear()
# The directory history can't be completely empty
self.dir_hist[:] = [py3compat.getcwd()]
if new_session:
if self.session_number:
self.end_session()
self.input_hist_parsed[:] = [""]
self.input_hist_raw[:] = [""]
self.new_session()
# ------------------------------
# Methods for retrieving history
# ------------------------------
def get_session_info(self, session=0):
"""Get info about a session.
Parameters
----------
session : int
Session number to retrieve. The current session is 0, and negative
numbers count back from current session, so -1 is the previous session.
Returns
-------
session_id : int
Session ID number
start : datetime
Timestamp for the start of the session.
end : datetime
Timestamp for the end of the session, or None if IPython crashed.
num_cmds : int
Number of commands run, or None if IPython crashed.
remark : unicode
A manually set description.
"""
if session <= 0:
session += self.session_number
return super(HistoryManager, self).get_session_info(session=session)
def _get_range_session(self, start=1, stop=None, raw=True, output=False):
"""Get input and output history from the current session. Called by
get_range, and takes similar parameters."""
input_hist = self.input_hist_raw if raw else self.input_hist_parsed
n = len(input_hist)
if start < 0:
start += n
if not stop or (stop > n):
stop = n
elif stop < 0:
stop += n
for i in range(start, stop):
if output:
line = (input_hist[i], self.output_hist_reprs.get(i))
else:
line = input_hist[i]
yield (0, i, line)
def get_range(self, session=0, start=1, stop=None, raw=True, output=False):
"""Retrieve input by session.
Parameters
----------
session : int
Session number to retrieve. The current session is 0, and negative
numbers count back from current session, so -1 is previous session.
start : int
First line to retrieve.
stop : int
End of line range (excluded from output itself). If None, retrieve
to the end of the session.
raw : bool
If True, return untranslated input
output : bool
If True, attempt to include output. This will be 'real' Python
objects for the current session, or text reprs from previous
sessions if db_log_output was enabled at the time. Where no output
is found, None is used.
Returns
-------
entries
An iterator over the desired lines. Each line is a 3-tuple, either
(session, line, input) if output is False, or
(session, line, (input, output)) if output is True.
"""
if session <= 0:
session += self.session_number
if session == self.session_number: # Current session
return self._get_range_session(start, stop, raw, output)
return super(HistoryManager, self).get_range(session, start, stop, raw,
output)
## ----------------------------
## Methods for storing history:
## ----------------------------
def store_inputs(self, line_num, source, source_raw=None):
"""Store source and raw input in history and create input cache
variables ``_i*``.
Parameters
----------
line_num : int
The prompt number of this input.
source : str
Python input.
source_raw : str, optional
If given, this is the raw input without any IPython transformations
applied to it. If not given, ``source`` is used.
"""
if source_raw is None:
source_raw = source
source = source.rstrip('\n')
source_raw = source_raw.rstrip('\n')
# do not store exit/quit commands
if self._exit_re.match(source_raw.strip()):
return
self.input_hist_parsed.append(source)
self.input_hist_raw.append(source_raw)
with self.db_input_cache_lock:
self.db_input_cache.append((line_num, source, source_raw))
# Trigger to flush cache and write to DB.
if len(self.db_input_cache) >= self.db_cache_size:
self.save_flag.set()
# update the auto _i variables
self._iii = self._ii
self._ii = self._i
self._i = self._i00
self._i00 = source_raw
# hackish access to user namespace to create _i1,_i2... dynamically
new_i = '_i%s' % line_num
to_main = {
'_i': self._i,
'_ii': self._ii,
'_iii': self._iii,
new_i: self._i00
}
if self.shell is not None:
self.shell.push(to_main, interactive=False)
def store_output(self, line_num):
"""If database output logging is enabled, this saves all the
outputs from the indicated prompt number to the database. It's
called by run_cell after code has been executed.
Parameters
----------
line_num : int
The line number from which to save outputs
"""
if (not self.db_log_output) or (line_num
not in self.output_hist_reprs):
return
output = self.output_hist_reprs[line_num]
with self.db_output_cache_lock:
self.db_output_cache.append((line_num, output))
if self.db_cache_size <= 1:
self.save_flag.set()
def _writeout_input_cache(self, conn):
with conn:
for line in self.db_input_cache:
conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
(self.session_number, ) + line)
def _writeout_output_cache(self, conn):
with conn:
for line in self.db_output_cache:
conn.execute("INSERT INTO output_history VALUES (?, ?, ?)",
(self.session_number, ) + line)
@needs_sqlite
def writeout_cache(self, conn=None):
"""Write any entries in the cache to the database."""
if conn is None:
conn = self.db
with self.db_input_cache_lock:
try:
self._writeout_input_cache(conn)
except sqlite3.IntegrityError:
self.new_session(conn)
print("ERROR! Session/line number was not unique in",
"database. History logging moved to new session",
self.session_number)
try:
# Try writing to the new session. If this fails, don't
# recurse
self._writeout_input_cache(conn)
except sqlite3.IntegrityError:
pass
finally:
self.db_input_cache = []
with self.db_output_cache_lock:
try:
self._writeout_output_cache(conn)
except sqlite3.IntegrityError:
print("!! Session/line number for output was not unique",
"in database. Output will not be stored.")
finally:
self.db_output_cache = []
class SubmitApp(BaseIPythonApplication):
name = Unicode(u'nbgrader-submit')
description = Unicode(u'Submit a completed assignment')
aliases = aliases
flags = flags
examples = examples
student = Unicode(os.environ['USER'])
timestamp = Unicode(str(datetime.datetime.now()))
assignment_directory = Unicode('.',
config=True,
help=dedent("""
The directory containing the assignment to be submitted.
"""))
assignment_name = Unicode('',
config=True,
help=dedent("""
The name of the assignment. Defaults to the name of the assignment
directory.
"""))
submissions_directory = Unicode("{}/.nbgrader/submissions".format(
os.environ['HOME']),
config=True,
help=dedent("""
The directory where the submission will be saved.
"""))
ignore = List([".ipynb_checkpoints", "*.pyc", "__pycache__"],
config=True,
help=dedent("""
List of file names or file globs to be ignored when creating the
submission.
"""))
def _ipython_dir_default(self):
d = os.path.join(os.environ["HOME"], ".nbgrader")
self._ipython_dir_changed('ipython_dir', d, d)
return d
# The classes added here determine how configuration will be documented
classes = List()
def _classes_default(self):
"""This has to be in a method, for TerminalIPythonApp to be available."""
return [ProfileDir]
def _log_level_default(self):
return logging.INFO
@catch_config_error
def initialize(self, argv=None):
if not os.path.exists(self.ipython_dir):
self.log.warning("Creating nbgrader directory: {}".format(
self.ipython_dir))
os.mkdir(self.ipython_dir)
if not os.path.exists(self.submissions_directory):
os.makedirs(self.submissions_directory)
super(SubmitApp, self).initialize(argv)
self.stage_default_config_file()
self.init_assignment_root()
if self.assignment_name == '':
self.assignment_name = os.path.basename(self.assignment_directory)
def init_assignment_root(self):
# Specifying notebooks on the command-line overrides (rather than adds)
# the notebook list
if self.extra_args:
patterns = self.extra_args
else:
patterns = [self.assignment_directory]
if len(patterns) == 0:
pass
elif len(patterns) == 1:
self.assignment_directory = patterns[0]
else:
raise ValueError("You must specify the name of a directory")
self.assignment_directory = os.path.abspath(self.assignment_directory)
if not os.path.isdir(self.assignment_directory):
raise ValueError("Path is not a directory: {}".format(
self.assignment_directory))
def _is_ignored(self, filename):
dirname = os.path.dirname(filename)
for expr in self.ignore:
globs = glob.glob(os.path.join(dirname, expr))
if filename in globs:
self.log.debug("Ignoring file: {}".format(filename))
return True
return False
def make_temp_copy(self):
"""Copy the submission to a temporary directory. Returns the path to the
temporary copy of the submission."""
# copy everything to a temporary directory
pth = os.path.join(self.tmpdir, self.assignment_name)
shutil.copytree(self.assignment_directory, pth)
os.chdir(self.tmpdir)
# get the user name, write it to file
with open(os.path.join(self.assignment_name, "user.txt"), "w") as fh:
fh.write(self.student)
# save the submission time
with open(os.path.join(self.assignment_name, "timestamp.txt"),
"w") as fh:
fh.write(self.timestamp)
return pth
def make_archive(self, path_to_submission):
"""Make a tarball of the submission. Returns the path to the created
archive."""
root, submission = os.path.split(path_to_submission)
os.chdir(root)
archive = os.path.join(self.tmpdir,
"{}.tar.gz".format(self.assignment_name))
tf = tarfile.open(archive, "w:gz")
for (dirname, dirnames, filenames) in os.walk(submission):
if self._is_ignored(dirname):
continue
for filename in filenames:
pth = os.path.join(dirname, filename)
if not self._is_ignored(pth):
self.log.debug("Adding '{}' to submission".format(pth))
tf.add(pth)
tf.close()
return archive
def submit(self, path_to_tarball):
"""Submit the assignment."""
archive = "{}.tar.gz".format(self.assignment_name)
target = os.path.join(self.submissions_directory, archive)
shutil.copy(path_to_tarball, target)
return target
def start(self):
super(SubmitApp, self).start()
self.tmpdir = tempfile.mkdtemp()
try:
path_to_copy = self.make_temp_copy()
path_to_tarball = self.make_archive(path_to_copy)
path_to_submission = self.submit(path_to_tarball)
except:
raise
else:
self.log.debug("Saved to '{}'".format(path_to_submission))
self.log.info("'{}' submitted by {} at {}".format(
self.assignment_name, self.student, self.timestamp))
finally:
shutil.rmtree(self.tmpdir)
