class FloatSliderWidget(_BoundedFloatWidget):
_view_name = Unicode('FloatSliderView', sync=True)
orientation = Enum([u'horizontal', u'vertical'],
u'horizontal',
help="Vertical or horizontal.",
sync=True)
readout = Bool(True,
help="Display the current value of the slider next to it.",
sync=True)
class IPEngineSetJob(WinHPCJob):
job_name = Str('IPEngineSet', config=False)
is_exclusive = Bool(False, config=True)
username = Str(find_username(), config=True)
priority = Enum(
('Lowest', 'BelowNormal', 'Normal', 'AboveNormal', 'Highest'),
default_value='Highest',
config=True)
requested_nodes = Str('', config=True)
project = Str('IPython', config=True)
class IntRangeSlider(_BoundedIntRange):
_view_name = Unicode('IntSliderView', sync=True)
orientation = Enum([u'horizontal', u'vertical'],
u'horizontal',
help="Vertical or horizontal.",
sync=True)
_range = Bool(True, help="Display a range selector", sync=True)
readout = Bool(True,
help="Display the current value of the slider next to it.",
sync=True)
class ExecuteReply(Reference):
execution_count = Integer()
status = Enum((u'ok', u'error'))
def check(self, d):
Reference.check(self, d)
if d['status'] == 'ok':
ExecuteReplyOkay().check(d)
elif d['status'] == 'error':
ExecuteReplyError().check(d)
class Renderer(DOMWidget):
_view_name = Unicode('RendererView', sync=True)
width = CInt(600, sync=True)
height = CInt(400, sync=True)
renderer_type = Enum(['webgl', 'canvas', 'auto'], 'auto', sync=True)
scene = Instance(Scene, sync=True)
camera = Instance(Camera, sync=True)
controls = List(Instance(Controls), sync=True)
effect = Instance(Effect, sync=True)
color = Color('black', sync=True)
class Parity(HasTraits):
value = Int(0)
parity = Enum(['odd', 'even'], default_value='even')
def _value_validate(self, value, trait):
if self.parity == 'even' and value % 2:
raise TraitError('Expected an even number')
if self.parity == 'odd' and (value % 2 == 0):
raise TraitError('Expected an odd number')
return value
class IntSlider(_BoundedInt):
"""Slider widget that represents a int bounded by a minimum and maximum value."""
_view_name = Unicode('IntSliderView', sync=True)
orientation = Enum([u'horizontal', u'vertical'],
u'horizontal',
help="Vertical or horizontal.",
sync=True)
_range = Bool(False, help="Display a range selector", sync=True)
readout = Bool(True,
help="Display the current value of the slider next to it.",
sync=True)
class Label(InstallerMixin, widgets.DOMWidget):
"""
Just some text...
"""
_view_name = Unicode('ipbs/LabelView', sync=True)
value = Unicode(sync=True)
html = Bool(False, sync=True)
lead = Bool(False, sync=True)
align = Enum(bs.Alignment, sync=True)
# 3.2?
transform = Enum(bs.Transformation, sync=True)
# bootstrap context color
context = Enum(bs.Context, default_value=bs.Context.default, sync=True)
def __init__(self, value=None, **kwargs):
if value is not None:
kwargs["value"] = value
super(Label, self).__init__(**kwargs)
class ExecuteReply(Reference):
execution_count = Integer()
status = Enum(('ok', 'error'))
def check(self, d):
for tst in Reference.check(self, d):
yield tst
if d['status'] == 'ok':
for tst in ExecuteReplyOkay().check(d):
yield tst
elif d['status'] == 'error':
for tst in ExecuteReplyError().check(d):
yield tst
class Material(Widget):
_view_name = Unicode('MaterialView', sync=True)
# id = TODO
name = Unicode('', sync=True)
side = Enum(['FrontSide', 'BackSide', 'DoubleSide'], 'DoubleSide', sync=True)
opacity = CFloat(1.0, sync=True)
transparent = Bool(False, sync=True)
blending = Enum(['NoBlending', 'NormalBlending', 'AdditiveBlending', 'SubtractiveBlending', 'MultiplyBlending',
'CustomBlending'], 'NormalBlending', sync=True)
blendSrc = Enum(['ZeroFactor', 'OneFactor', 'SrcColorFactor', 'OneMinusSrcColorFactor', 'SrcAlphaFactor',
'OneMinusSrcAlphaFactor', 'DstAlphaFactor', 'OneMinusDstAlphaFactor'], 'SrcAlphaFactor', sync=True)
blendDst = Enum(['DstColorFactor', 'OneMinusDstColorFactor', 'SrcAlphaSaturateFactor'], 'OneMinusDstColorFactor',
sync=True)
blendEquation = Enum(['AddEquation', 'SubtractEquation', 'ReverseSubtractEquation'], 'AddEquation', sync=True)
depthTest = Bool(True, sync=True)
depthWrite = Bool(True, sync=True)
polygonOffset = Bool(False, sync=True)
polygonOffsetFactor = CFloat(1.0, sync=True)
polygonOffsetUnits = CFloat(1.0, sync=True)
alphaTest = CFloat(1.0, sync=True)
overdraw = CFloat(1.0, sync=True)
visible = Bool(True, sync=True)
needsUpdate = Bool(True, sync=True)
class NBExtensionApp(BaseIPythonApplication):
"""Entry point for installing notebook extensions"""
description = """Install IPython notebook extensions
Usage
ipython install-nbextension file [more files, folders, archives or urls]
This copies files and/or folders into the IPython nbextensions directory.
If a URL is given, it will be downloaded.
If an archive is given, it will be extracted into nbextensions.
If the requested files are already up to date, no action is taken
unless --overwrite is specified.
"""
examples = """
ipython install-nbextension /path/to/d3.js /path/to/myextension
"""
aliases = aliases
flags = flags
overwrite = Bool(False,
config=True,
help="Force overwrite of existing files")
symlink = Bool(False,
config=True,
help="Create symlinks instead of copying files")
verbose = Enum((0, 1, 2),
default_value=1,
config=True,
help="Verbosity level")
def install_extensions(self):
install_nbextension(
self.extra_args,
overwrite=self.overwrite,
symlink=self.symlink,
verbose=self.verbose,
ipython_dir=self.ipython_dir,
)
def start(self):
if not self.extra_args:
nbext = pjoin(self.ipython_dir, u'nbextensions')
print("Notebook extensions in %s:" % nbext)
for ext in os.listdir(nbext):
print(u" %s" % ext)
else:
self.install_extensions()
class Panel(InstallerMixin, widgets.DOMWidget):
"""
A Panel, with optional header and footer.
"""
_view_name = Unicode('PanelView', sync=True)
_view_module = Unicode('nbextensions/ipbs/js/widget_panel', sync=True)
# bootstrap context color
context = Enum(Context, default_value=Context.default, sync=True)
# all of these take either a string or a list of widgets
title = Any(sync=True)
body = Any(sync=True)
heading = Any(sync=True)
footer = Any(sync=True)
class BinaryView(DOMWidget):
_view_module = Unicode('nbextensions/bitjet/bitjet', sync=True)
_view_name = Unicode('BinaryView', sync=True)
_model_module = Unicode('nbextensions/bitjet/bitjet', sync=True)
_model_name = Unicode('BinaryModel', sync=True)
datawidth = Int(2, sync=True)
if ndarray:
data = (Bytes(sync=True, to_json=b64encode_json)
| Instance(ndarray, sync=True, to_json=b64encode_json))
else:
data = Bytes(sync=True, to_json=b64encode_json)
blockwidth = Int(4, sync=True)
blockheight = Int(4, sync=True)
height = Int(200, sync=True)
width = Int(800, sync=True)
bits_per_block = Enum([1, 8], default_value=1, sync=True)
class DataTexture(Texture):
_view_name = Unicode('DataTextureView', sync=True)
data = List(CInt, sync=True)
format = Enum(['RGBAFormat', 'AlphaFormat', 'RGBFormat', 'LuminanceFormat', 'LuminanceAlphaFormat'],
'RGBAFormat', sync=True)
width = CInt(256, sync=True)
height = CInt(256, sync=True)
type = Enum(['UnsignedByteType', 'ByteType', 'ShortType', 'UnsignedShortType', 'IntType',
'UnsignedIntType', 'FloatType', 'UnsignedShort4444Type', 'UnsignedShort5551Type',
'UnsignedShort565Type'], 'UnsignedByteType', sync=True)
mapping = Enum(['UVMapping', 'CubeReflectionMapping', 'CubeRefractionMapping', 'SphericalReflectionMapping',
'SphericalRefractionMapping'], 'UVMapping', sync=True)
wrapS = Enum(['ClampToEdgeWrapping', 'RepeatWrapping', 'MirroredRepeatWrapping'], 'ClampToEdgeWrapping',
sync=True)
wrapT = Enum(['ClampToEdgeWrapping', 'RepeatWrapping', 'MirroredRepeatWrapping'], 'ClampToEdgeWrapping',
sync=True)
magFilter = Enum(['LinearFilter', 'NearestFilter'], 'LinearFilter', sync=True)
minFilter = Enum(['NearestFilter', 'NearestMipMapNearestFilter', 'NearestMipMapLinearFilter',
'LinearFilter', 'LinearMipMapNearestFilter'], 'NearestFilter', sync=True)
anisotropy = CInt(1, sync=True)
class NotebookExporter(Exporter):
"""Exports to an IPython notebook."""
nbformat_version = Enum(list(nbformat.versions),
default_value=nbformat.current_nbformat,
config=True,
help="""The nbformat version to write.
Use this to downgrade notebooks.
"""
)
def _file_extension_default(self):
return '.ipynb'
output_mimetype = 'application/json'
def from_notebook_node(self, nb, resources=None, **kw):
nb_copy, resources = super(NotebookExporter, self).from_notebook_node(nb, resources, **kw)
if self.nbformat_version != nb_copy.nbformat:
resources['output_suffix'] = '.v%i' % self.nbformat_version
else:
resources['output_suffix'] = '.nbconvert'
output = nbformat.writes(nb_copy, version=self.nbformat_version)
return output, resources
class Parameter(EmbeddedDocument):
name = Unicode()
param_type = Enum(default_value="String", values=param_types)
default = Any()
value = Any(db=False)
description = Unicode()
#parent_ref = ObjectIdTrait(allow_none = True)
def __eq__(self, other):
return hash(self) == hash(other)
def __hash__(self):
return hash(self.id)
def repr_name(self):
return self.name
def __str__(self):
return self.name
def __unicode__(self):
return self.name
class Application(SingletonConfigurable):
"""A singleton application with full configuration support."""
# The name of the application, will usually match the name of the command
# line application
name = Unicode(u'application')
# The description of the application that is printed at the beginning
# of the help.
description = Unicode(u'This is an application.')
# default section descriptions
option_description = Unicode(option_description)
keyvalue_description = Unicode(keyvalue_description)
subcommand_description = Unicode(subcommand_description)
# The usage and example string that goes at the end of the help string.
examples = Unicode()
# A sequence of Configurable subclasses whose config=True attributes will
# be exposed at the command line.
classes = List([])
# The version string of this application.
version = Unicode(u'0.0')
# the argv used to initialize the application
argv = List()
# The log level for the application
log_level = Enum(
(0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'),
default_value=logging.WARN,
config=True,
help="Set the log level by value or name.")
def _log_level_changed(self, name, old, new):
"""Adjust the log level when log_level is set."""
if isinstance(new, string_types):
new = getattr(logging, new)
self.log_level = new
self.log.setLevel(new)
_log_formatter_cls = LevelFormatter
log_datefmt = Unicode(
"%Y-%m-%d %H:%M:%S",
config=True,
help="The date format used by logging formatters for %(asctime)s")
def _log_datefmt_changed(self, name, old, new):
self._log_format_changed()
log_format = Unicode(
"[%(name)s]%(highlevel)s %(message)s",
config=True,
help="The Logging format template",
)
def _log_format_changed(self, name, old, new):
"""Change the log formatter when log_format is set."""
_log_handler = self.log.handlers[0]
_log_formatter = self._log_formatter_cls(fmt=new,
datefmt=self.log_datefmt)
_log_handler.setFormatter(_log_formatter)
log = Instance(logging.Logger)
def _log_default(self):
"""Start logging for this application.
The default is to log to stderr using a StreamHandler, if no default
handler already exists. The log level starts at logging.WARN, but this
can be adjusted by setting the ``log_level`` attribute.
"""
log = logging.getLogger(self.__class__.__name__)
log.setLevel(self.log_level)
log.propagate = False
_log = log # copied from Logger.hasHandlers() (new in Python 3.2)
while _log:
if _log.handlers:
return log
if not _log.propagate:
break
else:
_log = _log.parent
if sys.executable.endswith('pythonw.exe'):
# this should really go to a file, but file-logging is only
# hooked up in parallel applications
_log_handler = logging.StreamHandler(open(os.devnull, 'w'))
else:
_log_handler = logging.StreamHandler()
_log_formatter = self._log_formatter_cls(fmt=self.log_format,
datefmt=self.log_datefmt)
_log_handler.setFormatter(_log_formatter)
log.addHandler(_log_handler)
return log
# the alias map for configurables
aliases = Dict({'log-level': 'Application.log_level'})
# flags for loading Configurables or store_const style flags
# flags are loaded from this dict by '--key' flags
# this must be a dict of two-tuples, the first element being the Config/dict
# and the second being the help string for the flag
flags = Dict()
def _flags_changed(self, name, old, new):
"""ensure flags dict is valid"""
for key, value in iteritems(new):
assert len(value) == 2, "Bad flag: %r:%s" % (key, value)
assert isinstance(value[0],
(dict, Config)), "Bad flag: %r:%s" % (key, value)
assert isinstance(value[1],
string_types), "Bad flag: %r:%s" % (key, value)
# subcommands for launching other applications
# if this is not empty, this will be a parent Application
# this must be a dict of two-tuples,
# the first element being the application class/import string
# and the second being the help string for the subcommand
subcommands = Dict()
# parse_command_line will initialize a subapp, if requested
subapp = Instance('IPython.config.application.Application',
allow_none=True)
# extra command-line arguments that don't set config values
extra_args = List(Unicode)
def __init__(self, **kwargs):
SingletonConfigurable.__init__(self, **kwargs)
# Ensure my class is in self.classes, so my attributes appear in command line
# options and config files.
if self.__class__ not in self.classes:
self.classes.insert(0, self.__class__)
def _config_changed(self, name, old, new):
SingletonConfigurable._config_changed(self, name, old, new)
self.log.debug('Config changed:')
self.log.debug(repr(new))
@catch_config_error
def initialize(self, argv=None):
"""Do the basic steps to configure me.
Override in subclasses.
"""
self.parse_command_line(argv)
def start(self):
"""Start the app mainloop.
Override in subclasses.
"""
if self.subapp is not None:
return self.subapp.start()
def print_alias_help(self):
"""Print the alias part of the help."""
if not self.aliases:
return
lines = []
classdict = {}
for cls in self.classes:
# include all parents (up to, but excluding Configurable) in available names
for c in cls.mro()[:-3]:
classdict[c.__name__] = c
for alias, longname in iteritems(self.aliases):
classname, traitname = longname.split('.', 1)
cls = classdict[classname]
trait = cls.class_traits(config=True)[traitname]
help = cls.class_get_trait_help(trait).splitlines()
# reformat first line
help[0] = help[0].replace(longname, alias) + ' (%s)' % longname
if len(alias) == 1:
help[0] = help[0].replace('--%s=' % alias, '-%s ' % alias)
lines.extend(help)
# lines.append('')
print(os.linesep.join(lines))
def print_flag_help(self):
"""Print the flag part of the help."""
if not self.flags:
return
lines = []
for m, (cfg, help) in iteritems(self.flags):
prefix = '--' if len(m) > 1 else '-'
lines.append(prefix + m)
lines.append(indent(dedent(help.strip())))
# lines.append('')
print(os.linesep.join(lines))
def print_options(self):
if not self.flags and not self.aliases:
return
lines = ['Options']
lines.append('-' * len(lines[0]))
lines.append('')
for p in wrap_paragraphs(self.option_description):
lines.append(p)
lines.append('')
print(os.linesep.join(lines))
self.print_flag_help()
self.print_alias_help()
print()
def print_subcommands(self):
"""Print the subcommand part of the help."""
if not self.subcommands:
return
lines = ["Subcommands"]
lines.append('-' * len(lines[0]))
lines.append('')
for p in wrap_paragraphs(
self.subcommand_description.format(
app=os.path.basename(self.argv[0]))):
lines.append(p)
lines.append('')
for subc, (cls, help) in iteritems(self.subcommands):
lines.append(subc)
if help:
lines.append(indent(dedent(help.strip())))
lines.append('')
print(os.linesep.join(lines))
def print_help(self, classes=False):
"""Print the help for each Configurable class in self.classes.
If classes=False (the default), only flags and aliases are printed.
"""
self.print_description()
self.print_subcommands()
self.print_options()
if classes:
if self.classes:
print("Class parameters")
print("----------------")
print()
for p in wrap_paragraphs(self.keyvalue_description):
print(p)
print()
for cls in self.classes:
cls.class_print_help()
print()
else:
print("To see all available configurables, use `--help-all`")
print()
self.print_examples()
def print_description(self):
"""Print the application description."""
for p in wrap_paragraphs(self.description):
print(p)
print()
def print_examples(self):
"""Print usage and examples.
This usage string goes at the end of the command line help string
and should contain examples of the application's usage.
"""
if self.examples:
print("Examples")
print("--------")
print()
print(indent(dedent(self.examples.strip())))
print()
def print_version(self):
"""Print the version string."""
print(self.version)
def update_config(self, config):
"""Fire the traits events when the config is updated."""
# Save a copy of the current config.
newconfig = deepcopy(self.config)
# Merge the new config into the current one.
newconfig.merge(config)
# Save the combined config as self.config, which triggers the traits
# events.
self.config = newconfig
@catch_config_error
def initialize_subcommand(self, subc, argv=None):
"""Initialize a subcommand with argv."""
subapp, help = self.subcommands.get(subc)
if isinstance(subapp, string_types):
subapp = import_item(subapp)
# clear existing instances
self.__class__.clear_instance()
# instantiate
self.subapp = subapp.instance(config=self.config)
# and initialize subapp
self.subapp.initialize(argv)
def flatten_flags(self):
"""flatten flags and aliases, so cl-args override as expected.
This prevents issues such as an alias pointing to InteractiveShell,
but a config file setting the same trait in TerminalInteraciveShell
getting inappropriate priority over the command-line arg.
Only aliases with exactly one descendent in the class list
will be promoted.
"""
# build a tree of classes in our list that inherit from a particular
# it will be a dict by parent classname of classes in our list
# that are descendents
mro_tree = defaultdict(list)
for cls in self.classes:
clsname = cls.__name__
for parent in cls.mro()[1:-3]:
# exclude cls itself and Configurable,HasTraits,object
mro_tree[parent.__name__].append(clsname)
# flatten aliases, which have the form:
# { 'alias' : 'Class.trait' }
aliases = {}
for alias, cls_trait in iteritems(self.aliases):
cls, trait = cls_trait.split('.', 1)
children = mro_tree[cls]
if len(children) == 1:
# exactly one descendent, promote alias
cls = children[0]
aliases[alias] = '.'.join([cls, trait])
# flatten flags, which are of the form:
# { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
flags = {}
for key, (flagdict, help) in iteritems(self.flags):
newflag = {}
for cls, subdict in iteritems(flagdict):
children = mro_tree[cls]
# exactly one descendent, promote flag section
if len(children) == 1:
cls = children[0]
newflag[cls] = subdict
flags[key] = (newflag, help)
return flags, aliases
@catch_config_error
def parse_command_line(self, argv=None):
"""Parse the command line arguments."""
argv = sys.argv[1:] if argv is None else argv
self.argv = [py3compat.cast_unicode(arg) for arg in argv]
if argv and argv[0] == 'help':
# turn `ipython help notebook` into `ipython notebook -h`
argv = argv[1:] + ['-h']
if self.subcommands and len(argv) > 0:
# we have subcommands, and one may have been specified
subc, subargv = argv[0], argv[1:]
if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
# it's a subcommand, and *not* a flag or class parameter
return self.initialize_subcommand(subc, subargv)
# Arguments after a '--' argument are for the script IPython may be
# about to run, not IPython iteslf. For arguments parsed here (help and
# version), we want to only search the arguments up to the first
# occurrence of '--', which we're calling interpreted_argv.
try:
interpreted_argv = argv[:argv.index('--')]
except ValueError:
interpreted_argv = argv
if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')):
self.print_help('--help-all' in interpreted_argv)
self.exit(0)
if '--version' in interpreted_argv or '-V' in interpreted_argv:
self.print_version()
self.exit(0)
# flatten flags&aliases, so cl-args get appropriate priority:
flags, aliases = self.flatten_flags()
loader = KVArgParseConfigLoader(argv=argv,
aliases=aliases,
flags=flags,
log=self.log)
config = loader.load_config()
self.update_config(config)
# store unparsed args in extra_args
self.extra_args = loader.extra_args
@classmethod
def _load_config_files(cls, basefilename, path=None, log=None):
"""Load config files (py,json) by filename and path.
yield each config object in turn.
"""
pyloader = PyFileConfigLoader(basefilename + '.py', path=path, log=log)
jsonloader = JSONFileConfigLoader(basefilename + '.json',
path=path,
log=log)
config = None
for loader in [pyloader, jsonloader]:
try:
config = loader.load_config()
except ConfigFileNotFound:
pass
except Exception:
# try to get the full filename, but it will be empty in the
# unlikely event that the error raised before filefind finished
filename = loader.full_filename or basefilename
# problem while running the file
if log:
log.error("Exception while loading config file %s",
filename,
exc_info=True)
else:
if log:
log.debug("Loaded config file: %s", loader.full_filename)
if config:
yield config
raise StopIteration
@catch_config_error
def load_config_file(self, filename, path=None):
"""Load config files by filename and path."""
filename, ext = os.path.splitext(filename)
for config in self._load_config_files(filename,
path=path,
log=self.log):
self.update_config(config)
def generate_config_file(self):
"""generate default config file from Configurables"""
lines = ["# Configuration file for %s." % self.name]
lines.append('')
lines.append('c = get_config()')
lines.append('')
for cls in self.classes:
lines.append(cls.class_config_section())
return '\n'.join(lines)
def exit(self, exit_status=0):
self.log.debug("Exiting application: %s" % self.name)
sys.exit(exit_status)
@classmethod
def launch_instance(cls, argv=None, **kwargs):
"""Launch a global instance of this Application
If a global instance already exists, this reinitializes and starts it
"""
app = cls.instance(**kwargs)
app.initialize(argv)
app.start()
class InProcessKernel(IPythonKernel):
#-------------------------------------------------------------------------
# InProcessKernel interface
#-------------------------------------------------------------------------
# The frontends connected to this kernel.
frontends = List(
Instance('IPython.kernel.inprocess.client.InProcessKernelClient'))
# The GUI environment that the kernel is running under. This need not be
# specified for the normal operation for the kernel, but is required for
# IPython's GUI support (including pylab). The default is 'inline' because
# it is safe under all GUI toolkits.
gui = Enum(('tk', 'gtk', 'wx', 'qt', 'qt4', 'inline'),
default_value='inline')
raw_input_str = Any()
stdout = Any()
stderr = Any()
#-------------------------------------------------------------------------
# Kernel interface
#-------------------------------------------------------------------------
shell_class = Type()
shell_streams = List()
control_stream = Any()
iopub_socket = Instance(DummySocket, ())
stdin_socket = Instance(DummySocket, ())
def __init__(self, **traits):
# When an InteractiveShell is instantiated by our base class, it binds
# the current values of sys.stdout and sys.stderr.
with self._redirected_io():
super(InProcessKernel, self).__init__(**traits)
self.iopub_socket.on_trait_change(self._io_dispatch, 'message_sent')
self.shell.kernel = self
def execute_request(self, stream, ident, parent):
""" Override for temporary IO redirection. """
with self._redirected_io():
super(InProcessKernel, self).execute_request(stream, ident, parent)
def start(self):
""" Override registration of dispatchers for streams. """
self.shell.exit_now = False
def _abort_queue(self, stream):
""" The in-process kernel doesn't abort requests. """
pass
def _input_request(self, prompt, ident, parent, password=False):
# Flush output before making the request.
self.raw_input_str = None
sys.stderr.flush()
sys.stdout.flush()
# Send the input request.
content = json_clean(dict(prompt=prompt, password=password))
msg = self.session.msg(u'input_request', content, parent)
for frontend in self.frontends:
if frontend.session.session == parent['header']['session']:
frontend.stdin_channel.call_handlers(msg)
break
else:
logging.error('No frontend found for raw_input request')
return str()
# Await a response.
while self.raw_input_str is None:
frontend.stdin_channel.process_events()
return self.raw_input_str
#-------------------------------------------------------------------------
# Protected interface
#-------------------------------------------------------------------------
@contextmanager
def _redirected_io(self):
""" Temporarily redirect IO to the kernel.
"""
sys_stdout, sys_stderr = sys.stdout, sys.stderr
sys.stdout, sys.stderr = self.stdout, self.stderr
yield
sys.stdout, sys.stderr = sys_stdout, sys_stderr
#------ Trait change handlers --------------------------------------------
def _io_dispatch(self):
""" Called when a message is sent to the IO socket.
"""
ident, msg = self.session.recv(self.iopub_socket, copy=False)
for frontend in self.frontends:
frontend.iopub_channel.call_handlers(msg)
#------ Trait initializers -----------------------------------------------
def _log_default(self):
return logging.getLogger(__name__)
def _session_default(self):
from IPython.kernel.zmq.session import Session
return Session(parent=self, key=b'')
def _shell_class_default(self):
return InProcessInteractiveShell
def _stdout_default(self):
from IPython.kernel.zmq.iostream import OutStream
return OutStream(self.session,
self.iopub_socket,
u'stdout',
pipe=False)
def _stderr_default(self):
from IPython.kernel.zmq.iostream import OutStream
return OutStream(self.session,
self.iopub_socket,
u'stderr',
pipe=False)
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
default_config_file_name = 'ipython_notebook_config.py'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = IPythonConsoleApp.classes + [
MappingKernelManager, NotebookManager, FileNotebookManager
]
flags = Dict(flags)
aliases = Dict(aliases)
kernel_argv = List(Unicode)
log_level = Enum(
(0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'),
default_value=logging.INFO,
config=True,
help="Set the log level by value or name.")
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('')
# Network related information.
ip = Unicode(LOCALHOST,
config=True,
help="The IP address the notebook server will listen on.")
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888,
config=True,
help="The port the notebook server will listen on.")
port_retries = Integer(
50,
config=True,
help=
"The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(
u'',
config=True,
help="""The full path to an SSL/TLS certificate file.""")
keyfile = Unicode(
u'',
config=True,
help="""The full path to a private key file for usage with SSL/TLS.""")
password = Unicode(u'',
config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
""")
open_browser = Bool(True,
config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'',
config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
read_only = Bool(False,
config=True,
help="Whether to prevent editing/execution of notebooks.")
use_less = Bool(False,
config=True,
help="""Wether to use Browser Side less-css parsing
instead of compiled css version in templates that allows
it. This is mainly convenient when working on the less
file to avoid a build step, or if user want to overwrite
some of the less variables without having to recompile
everything.
You will need to install the less.js component in the static directory
either in the source tree or in your profile folder.
""")
webapp_settings = Dict(
config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
enable_mathjax = Bool(
True,
config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
""")
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_project_url = Unicode('/',
config=True,
help='''The base URL for the notebook server.
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_project_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_project_url = '/' + new
elif not new.endswith('/'):
self.base_project_url = new + '/'
base_kernel_url = Unicode('/',
config=True,
help='''The base URL for the kernel server
Leading and trailing slashes can be omitted,
and will automatically be added.
''')
def _base_kernel_url_changed(self, name, old, new):
if not new.startswith('/'):
self.base_kernel_url = '/' + new
elif not new.endswith('/'):
self.base_kernel_url = new + '/'
websocket_host = Unicode("",
config=True,
help="""The hostname for the websocket server.""")
extra_static_paths = List(
Unicode,
config=True,
help="""Extra paths to search for serving static files.
This allows adding javascript/css to be available from the notebook server machine,
or overriding individual files in the IPython""")
def _extra_static_paths_default(self):
return [os.path.join(self.profile_dir.location, 'static')]
@property
def static_file_path(self):
"""return extra paths + the default location"""
return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""")
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_url_prefix = self.webapp_settings.get("static_url_prefix",
"/static/")
try:
mathjax = filefind(os.path.join('mathjax', 'MathJax.js'),
self.static_file_path)
except IOError:
if self.certfile:
# HTTPS: load from Rackspace CDN, because SSL certificate requires it
base = u"https://c328740.ssl.cf1.rackcdn.com"
else:
base = u"http://cdn.mathjax.org"
url = base + u"/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
else:
self.log.info("Using local MathJax from %s" % mathjax)
return static_url_prefix + u"mathjax/MathJax.js"
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
notebook_manager_class = DottedObjectName(
'IPython.frontend.html.notebook.filenbmanager.FileNotebookManager',
config=True,
help='The notebook manager class to use.')
trust_xheaders = Bool(
False,
config=True,
help=
("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
"sent by the upstream reverse proxy. Neccesary if the proxy handles SSL"
))
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if argv is None:
argv = sys.argv[1:]
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" %
self.name)
if self.extra_args:
f = os.path.abspath(self.extra_args[0])
if os.path.isdir(f):
nbdir = f
else:
self.file_to_run = f
nbdir = os.path.dirname(f)
self.config.NotebookManager.notebook_dir = nbdir
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_manager = MappingKernelManager(
config=self.config,
log=self.log,
kernel_argv=self.kernel_argv,
connection_dir=self.profile_dir.security_dir,
)
kls = import_item(self.notebook_manager_class)
self.notebook_manager = kls(config=self.config, log=self.log)
self.notebook_manager.load_notebook_names()
self.cluster_manager = ClusterManager(config=self.config, log=self.log)
self.cluster_manager.update_profiles()
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(self, self.kernel_manager,
self.notebook_manager,
self.cluster_manager, self.log,
self.base_project_url,
self.webapp_settings)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app,
ssl_options=ssl_options,
xheaders=self.trust_xheaders)
if not self.ip:
warning = "WARNING: The notebook server is listening on all IP addresses"
if ssl_options is None:
self.log.critical(warning + " and not using encryption. This"
"is not recommended.")
if not self.password and not self.read_only:
self.log.critical(
warning + "and not using authentication."
"This is highly insecure and not recommended.")
success = None
for port in random_ports(self.port, self.port_retries + 1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
# XXX: remove the e.errno == -9 block when we require
# tornado >= 3.0
if e.errno == -9 and tornado.version_info[0] < 3:
# The flags passed to socket.getaddrinfo from
# tornado.netutils.bind_sockets can cause "gaierror:
# [Errno -9] Address family for hostname not supported"
# when the interface is not associated, for example.
# Changing the flags to exclude socket.AI_ADDRCONFIG does
# not cause this error, but the only way to do this is to
# monkeypatch socket to remove the AI_ADDRCONFIG attribute
saved_AI_ADDRCONFIG = socket.AI_ADDRCONFIG
self.log.warn('Monkeypatching socket to fix tornado bug')
del (socket.AI_ADDRCONFIG)
try:
# retry the tornado call without AI_ADDRCONFIG flags
self.http_server.listen(port, self.ip)
except socket.error as e2:
e = e2
else:
self.port = port
success = True
break
# restore the monekypatch
socket.AI_ADDRCONFIG = saved_AI_ADDRCONFIG
if e.errno != errno.EADDRINUSE:
raise
self.log.info(
'The port %i is already in use, trying another random port.'
% port)
else:
self.port = port
success = True
break
if not success:
self.log.critical(
'ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
def init_signal(self):
if not sys.platform.startswith('win'):
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
if hasattr(signal, 'SIGUSR1'):
# Windows doesn't support SIGUSR1
signal.signal(signal.SIGUSR1, self._signal_info)
if hasattr(signal, 'SIGINFO'):
# only on BSD-based systems
signal.signal(signal.SIGINFO, self._signal_info)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
# FIXME: remove this delay when pyzmq dependency is >= 2.1.11
time.sleep(0.1)
info = self.log.info
info('interrupted')
print self.notebook_info()
sys.stdout.write("Shutdown this notebook server (y/[n])? ")
sys.stdout.flush()
r, w, x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y'):
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.instance().stop()
return
else:
print "No answer for 5s:",
print "resuming operation..."
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.instance().stop()
def _signal_info(self, sig, frame):
print self.notebook_info()
@catch_config_error
def initialize(self, argv=None):
self.init_logging()
super(NotebookApp, self).initialize(argv)
self.init_configurables()
self.init_webapp()
self.init_signal()
def cleanup_kernels(self):
"""Shutdown all kernels.
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
self.kernel_manager.shutdown_all()
def notebook_info(self):
"Return the current working directory and the server url information"
mgr_info = self.notebook_manager.info_string() + "\n"
return mgr_info + "The IPython Notebook is running at: %s" % self._url
def start(self):
""" Start the IPython Notebok server app, after initialization
This method takes no arguments so all configuration and initialization
must be done prior to calling this method."""
ip = self.ip if self.ip else '[all ip addresses on your system]'
proto = 'https' if self.certfile else 'http'
info = self.log.info
self._url = "%s://%s:%i%s" % (proto, ip, self.port,
self.base_project_url)
for line in self.notebook_info().split("\n"):
info(line)
info("Use Control-C to stop this server and shut down all kernels.")
if self.open_browser or self.file_to_run:
ip = self.ip or LOCALHOST
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
name, _ = os.path.splitext(os.path.basename(self.file_to_run))
url = self.notebook_manager.rev_mapping.get(name, '')
else:
url = ''
if browser:
b = lambda: browser.open("%s://%s:%i%s%s" % (
proto, ip, self.port, self.base_project_url, url),
new=2)
threading.Thread(target=b).start()
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
class ZMQTerminalInteractiveShell(TerminalInteractiveShell):
"""A subclass of TerminalInteractiveShell that uses the 0MQ kernel"""
_executing = False
_execution_state = Unicode('')
kernel_timeout = Float(
60,
config=True,
help="""Timeout for giving up on a kernel (in seconds).
On first connect and restart, the console tests whether the
kernel is running and responsive by sending kernel_info_requests.
This sets the timeout in seconds for how long the kernel can take
before being presumed dead.
""")
image_handler = Enum(('PIL', 'stream', 'tempfile', 'callable'),
config=True,
help="""
Handler for image type output. This is useful, for example,
when connecting to the kernel in which pylab inline backend is
activated. There are four handlers defined. 'PIL': Use
Python Imaging Library to popup image; 'stream': Use an
external program to show the image. Image will be fed into
the STDIN of the program. You will need to configure
`stream_image_handler`; 'tempfile': Use an external program to
show the image. Image will be saved in a temporally file and
the program is called with the temporally file. You will need
to configure `tempfile_image_handler`; 'callable': You can set
any Python callable which is called with the image data. You
will need to configure `callable_image_handler`.
""")
stream_image_handler = List(config=True,
help="""
Command to invoke an image viewer program when you are using
'stream' image handler. This option is a list of string where
the first element is the command itself and reminders are the
options for the command. Raw image data is given as STDIN to
the program.
""")
tempfile_image_handler = List(config=True,
help="""
Command to invoke an image viewer program when you are using
'tempfile' image handler. This option is a list of string
where the first element is the command itself and reminders
are the options for the command. You can use {file} and
{format} in the string to represent the location of the
generated image file and image format.
""")
callable_image_handler = Any(config=True,
help="""
Callable object called via 'callable' image handler with one
argument, `data`, which is `msg["content"]["data"]` where
`msg` is the message from iopub channel. For exmaple, you can
find base64 encoded PNG data as `data['image/png']`.
""")
mime_preference = List(
default_value=['image/png', 'image/jpeg', 'image/svg+xml'],
config=True,
allow_none=False,
help="""
Preferred object representation MIME type in order. First
matched MIME type will be used.
""")
manager = Instance('IPython.kernel.KernelManager')
client = Instance('IPython.kernel.KernelClient')
def _client_changed(self, name, old, new):
self.session_id = new.session.session
session_id = Unicode()
def init_completer(self):
"""Initialize the completion machinery.
This creates completion machinery that can be used by client code,
either interactively in-process (typically triggered by the readline
library), programmatically (such as in test suites) or out-of-process
(typically over the network by remote frontends).
"""
from IPython.core.completerlib import (module_completer,
magic_run_completer,
cd_completer)
self.Completer = ZMQCompleter(self, self.client, config=self.config)
self.set_hook('complete_command', module_completer, str_key='import')
self.set_hook('complete_command', module_completer, str_key='from')
self.set_hook('complete_command', magic_run_completer, str_key='%run')
self.set_hook('complete_command', cd_completer, str_key='%cd')
# Only configure readline if we truly are using readline. IPython can
# do tab-completion over the network, in GUIs, etc, where readline
# itself may be absent
if self.has_readline:
self.set_readline_completer()
def ask_exit(self):
super(ZMQTerminalInteractiveShell, self).ask_exit()
if self.exit_now and self.manager:
self.client.shutdown()
def run_cell(self, cell, store_history=True):
"""Run a complete IPython cell.
Parameters
----------
cell : str
The code (including IPython code such as %magic functions) to run.
store_history : bool
If True, the raw and translated cell will be stored in IPython's
history. For user code calling back into IPython's machinery, this
should be set to False.
"""
if (not cell) or cell.isspace():
return
if cell.strip() == 'exit':
# explicitly handle 'exit' command
return self.ask_exit()
# flush stale replies, which could have been ignored, due to missed heartbeats
while self.client.shell_channel.msg_ready():
self.client.shell_channel.get_msg()
# shell_channel.execute takes 'hidden', which is the inverse of store_hist
msg_id = self.client.shell_channel.execute(cell, not store_history)
# first thing is wait for any side effects (output, stdin, etc.)
self._executing = True
self._execution_state = "busy"
while self._execution_state != 'idle' and self.client.is_alive():
try:
self.handle_stdin_request(msg_id, timeout=0.05)
except Empty:
# display intermediate print statements, etc.
self.handle_iopub(msg_id)
pass
# after all of that is done, wait for the execute reply
while self.client.is_alive():
try:
self.handle_execute_reply(msg_id, timeout=0.05)
except Empty:
pass
else:
break
self._executing = False
#-----------------
# message handlers
#-----------------
def handle_execute_reply(self, msg_id, timeout=None):
msg = self.client.shell_channel.get_msg(block=False, timeout=timeout)
if msg["parent_header"].get("msg_id", None) == msg_id:
self.handle_iopub(msg_id)
content = msg["content"]
status = content['status']
if status == 'aborted':
self.write('Aborted\n')
return
elif status == 'ok':
# print execution payloads as well:
for item in content["payload"]:
text = item.get('text', None)
if text:
page.page(text)
elif status == 'error':
for frame in content["traceback"]:
print(frame, file=io.stderr)
self.execution_count = int(content["execution_count"] + 1)
def handle_iopub(self, msg_id):
""" Method to process subscribe channel's messages
This method consumes and processes messages on the IOPub channel,
such as stdout, stderr, pyout and status.
It only displays output that is caused by the given msg_id
"""
while self.client.iopub_channel.msg_ready():
sub_msg = self.client.iopub_channel.get_msg()
msg_type = sub_msg['header']['msg_type']
parent = sub_msg["parent_header"]
if (not parent) or msg_id == parent['msg_id']:
if msg_type == 'status':
state = self._execution_state = sub_msg["content"][
"execution_state"]
# idle messages mean an individual sequence is complete,
# so break out of consumption to allow other things to take over.
if state == 'idle':
break
elif msg_type == 'stream':
if sub_msg["content"]["name"] == "stdout":
print(sub_msg["content"]["data"],
file=io.stdout,
end="")
io.stdout.flush()
elif sub_msg["content"]["name"] == "stderr":
print(sub_msg["content"]["data"],
file=io.stderr,
end="")
io.stderr.flush()
elif msg_type == 'pyout':
self.execution_count = int(
sub_msg["content"]["execution_count"])
format_dict = sub_msg["content"]["data"]
self.handle_rich_data(format_dict)
# taken from DisplayHook.__call__:
hook = self.displayhook
hook.start_displayhook()
hook.write_output_prompt()
hook.write_format_data(format_dict)
hook.log_output(format_dict)
hook.finish_displayhook()
elif msg_type == 'display_data':
data = sub_msg["content"]["data"]
handled = self.handle_rich_data(data)
if not handled:
# if it was an image, we handled it by now
if 'text/plain' in data:
print(data['text/plain'])
_imagemime = {
'image/png': 'png',
'image/jpeg': 'jpeg',
'image/svg+xml': 'svg',
}
def handle_rich_data(self, data):
for mime in self.mime_preference:
if mime in data and mime in self._imagemime:
self.handle_image(data, mime)
return True
def handle_image(self, data, mime):
handler = getattr(self, 'handle_image_{0}'.format(self.image_handler),
None)
if handler:
handler(data, mime)
def handle_image_PIL(self, data, mime):
if mime not in ('image/png', 'image/jpeg'):
return
import PIL.Image
raw = base64.decodestring(data[mime].encode('ascii'))
img = PIL.Image.open(BytesIO(raw))
img.show()
def handle_image_stream(self, data, mime):
raw = base64.decodestring(data[mime].encode('ascii'))
imageformat = self._imagemime[mime]
fmt = dict(format=imageformat)
args = [s.format(**fmt) for s in self.stream_image_handler]
with open(os.devnull, 'w') as devnull:
proc = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=devnull,
stderr=devnull)
proc.communicate(raw)
def handle_image_tempfile(self, data, mime):
raw = base64.decodestring(data[mime].encode('ascii'))
imageformat = self._imagemime[mime]
filename = 'tmp.{0}'.format(imageformat)
with NamedFileInTemporaryDirectory(filename) as f, \
open(os.devnull, 'w') as devnull:
f.write(raw)
f.flush()
fmt = dict(file=f.name, format=imageformat)
args = [s.format(**fmt) for s in self.tempfile_image_handler]
subprocess.call(args, stdout=devnull, stderr=devnull)
def handle_image_callable(self, data, mime):
self.callable_image_handler(data)
def handle_stdin_request(self, msg_id, timeout=0.1):
""" Method to capture raw_input
"""
msg_rep = self.client.stdin_channel.get_msg(timeout=timeout)
# in case any iopub came while we were waiting:
self.handle_iopub(msg_id)
if msg_id == msg_rep["parent_header"].get("msg_id"):
# wrap SIGINT handler
real_handler = signal.getsignal(signal.SIGINT)
def double_int(sig, frame):
# call real handler (forwards sigint to kernel),
# then raise local interrupt, stopping local raw_input
real_handler(sig, frame)
raise KeyboardInterrupt
signal.signal(signal.SIGINT, double_int)
try:
raw_data = input(msg_rep["content"]["prompt"])
except EOFError:
# turn EOFError into EOF character
raw_data = '\x04'
except KeyboardInterrupt:
sys.stdout.write('\n')
return
finally:
# restore SIGINT handler
signal.signal(signal.SIGINT, real_handler)
# only send stdin reply if there *was not* another request
# or execution finished while we were reading.
if not (self.client.stdin_channel.msg_ready()
or self.client.shell_channel.msg_ready()):
self.client.stdin_channel.input(raw_data)
def mainloop(self, display_banner=False):
while True:
try:
self.interact(display_banner=display_banner)
#self.interact_with_readline()
# XXX for testing of a readline-decoupled repl loop, call
# interact_with_readline above
break
except KeyboardInterrupt:
# this should not be necessary, but KeyboardInterrupt
# handling seems rather unpredictable...
self.write("\nKeyboardInterrupt in interact()\n")
def wait_for_kernel(self, timeout=None):
"""method to wait for a kernel to be ready"""
tic = time.time()
self.client.hb_channel.unpause()
while True:
msg_id = self.client.kernel_info()
reply = None
while True:
try:
reply = self.client.get_shell_msg(timeout=1)
except Empty:
break
else:
if reply['parent_header'].get('msg_id') == msg_id:
return True
if timeout is not None \
and (time.time() - tic) > timeout \
and not self.client.hb_channel.is_beating():
# heart failed
return False
return True
def interact(self, display_banner=None):
"""Closely emulate the interactive Python console."""
# batch run -> do not interact
if self.exit_now:
return
if display_banner is None:
display_banner = self.display_banner
if isinstance(display_banner, string_types):
self.show_banner(display_banner)
elif display_banner:
self.show_banner()
more = False
# run a non-empty no-op, so that we don't get a prompt until
# we know the kernel is ready. This keeps the connection
# message above the first prompt.
if not self.wait_for_kernel(self.kernel_timeout):
error("Kernel did not respond\n")
return
if self.has_readline:
self.readline_startup_hook(self.pre_readline)
hlen_b4_cell = self.readline.get_current_history_length()
else:
hlen_b4_cell = 0
# exit_now is set by a call to %Exit or %Quit, through the
# ask_exit callback.
while not self.exit_now:
if not self.client.is_alive():
# kernel died, prompt for action or exit
action = "restart" if self.manager else "wait for restart"
ans = self.ask_yes_no("kernel died, %s ([y]/n)?" % action,
default='y')
if ans:
if self.manager:
self.manager.restart_kernel(True)
self.wait_for_kernel(self.kernel_timeout)
else:
self.exit_now = True
continue
try:
# protect prompt block from KeyboardInterrupt
# when sitting on ctrl-C
self.hooks.pre_prompt_hook()
if more:
try:
prompt = self.prompt_manager.render('in2')
except Exception:
self.showtraceback()
if self.autoindent:
self.rl_do_indent = True
else:
try:
prompt = self.separate_in + self.prompt_manager.render(
'in')
except Exception:
self.showtraceback()
line = self.raw_input(prompt)
if self.exit_now:
# quick exit on sys.std[in|out] close
break
if self.autoindent:
self.rl_do_indent = False
except KeyboardInterrupt:
#double-guard against keyboardinterrupts during kbdint handling
try:
self.write('\nKeyboardInterrupt\n')
source_raw = self.input_splitter.raw_reset()
hlen_b4_cell = self._replace_rlhist_multiline(
source_raw, hlen_b4_cell)
more = False
except KeyboardInterrupt:
pass
except EOFError:
if self.autoindent:
self.rl_do_indent = False
if self.has_readline:
self.readline_startup_hook(None)
self.write('\n')
self.exit()
except bdb.BdbQuit:
warn(
'The Python debugger has exited with a BdbQuit exception.\n'
'Because of how pdb handles the stack, it is impossible\n'
'for IPython to properly format this particular exception.\n'
'IPython will resume normal operation.')
except:
# exceptions here are VERY RARE, but they can be triggered
# asynchronously by signal handlers, for example.
self.showtraceback()
else:
try:
self.input_splitter.push(line)
more = self.input_splitter.push_accepts_more()
except SyntaxError:
# Run the code directly - run_cell takes care of displaying
# the exception.
more = False
if (self.SyntaxTB.last_syntax_error and self.autoedit_syntax):
self.edit_syntax_error()
if not more:
source_raw = self.input_splitter.raw_reset()
hlen_b4_cell = self._replace_rlhist_multiline(
source_raw, hlen_b4_cell)
self.run_cell(source_raw)
# Turn off the exit flag, so the mainloop can be restarted if desired
self.exit_now = False
class Status(Reference):
execution_state = Enum((u'busy', u'idle', u'starting'))
class SpectraModel(HTML, Box):
"""
A notional "complex widget" that knows how to redraw itself when key
properties change.
"""
# CONSTANTS (These are not traits)
classname = Unicode("btn btn-success", sync=True)
title = Unicode("Popover Test", sync=True)
CONTENT = Unicode(
"""Lovely popover :D. Color in green using class btn btn-success""",
sync=True)
html = Bool(sync=True)
DONT_DRAW = re.compile(
r'^(_.+|value|keys|comm|children|visible|parent|log|config|msg_throttle)$'
)
SPECUNITS = aunps_glass().specunits()
VARUNITS = aunps_glass().varunits()
NORMUNITS = NUdic
SPECUNITS_REV = OrderedDict((v, k) for k, v in SPECUNITS.items())
VARUNITS_REV = OrderedDict((v, k) for k, v in VARUNITS.items())
NORMUNITS_REV = OrderedDict((v, k) for k, v in NORMUNITS.items())
COLORS = ["b", "g", "r", "y", "k"]
COLORMAPS = sorted(m for m in plt.cm.datad if not m.endswith("_r"))
SLIDER_STEPS = Float(25)
# IO traitlets
load_spec = Bool(False, sync=True)
load_file = Bool(True, sync=True) #
file_name = Unicode("<Import Variable Name>", sync=True)
save_spec = Bool(False, sync=True)
save_spec_as = Unicode("<Export Name>", sync=True)
inbox = Bool(False, sync=True)
outbox = Bool(False, sync=True)
# Spectra traits
spec = Instance(Spectra)
testdataset = Unicode('<Dataset>', sync=True)
spec_modified = Instance(Spectra)
# Plotting Traits
figwidth = Float(6.5)
figheight = Float(6.5)
interactive = Bool(False, sync=True)
colorbar = Bool(False, sync=True)
autoupdate = Bool(True, sync=True)
colormap = Enum(COLORMAPS, sync=True)
color = Enum(COLORS, default_value='k', sync=True)
advancedbox = Bool(False, sync=True)
cmapbox = Bool(False, sync=True)
colorbox = Bool(False, sync=True)
kind = Enum(PLOTPARSER.keys(), default_value='spec', sync=True)
selectlines = Bool(False, sync=True)
# Units
spec_unit = Enum(SPECUNITS.values(), sync=True)
var_unit = Enum(VARUNITS.values(), sync=True)
iunit = Unicode
norm_unit = Enum(NORMUNITS.values(), sync=True)
# Message/warnings
message = Unicode
# Sampling/slicing
#specslice_axis = Enum([0,1], default_value=0, sync=True)
specslice_position_start = Float(sync=True)
specslice_position_end = Float(sync=True)
specslider_start = Float(sync=True)
specslider_end = Float(sync=True)
specstep = Float(sync=True)
specspacing = Int(1, sync=True)
timeslice_position_start = Float(sync=True)
timeslice_position_end = Float(sync=True)
timeslider_start = Float(sync=True)
timeslider_end = Float(sync=True)
timestep = Float(sync=True)
timespacing = Int(1, sync=True)
specbox = Bool(False, sync=True)
timebox = Bool(False, sync=True)
# User Defined Function
user_f = Unicode(sync=True)
def __init__(self, *args, **kwargs):
# Initialize traits (_spec_changed calls initial draw)
super(SpectraModel, self).__init__(*args, **kwargs)
self._dom_classes += ("col-xs-9", )
# DEFAULTS
# --------
def _spec_default(self):
return getattr(skspec.data, 'aunps_water')()
def _colormap_default(self):
return pvconf.CMAP_1DSPECPLOT #Use skspec config default (red/blue map)
# Events
# ------
def _spec_changed(self, name, old, new):
"""Overall spectrum changes; triggers most events."""
# Leave this at this position in loop
self.spec_modified = self.spec
# --------------
self._FREEZE = True #pause draws/slicing
# Units
self.spec_unit = self.spec.full_specunit
self.var_unit = self.spec.full_varunit
self.norm_unit = self.spec.full_norm
self.iunit = self.spec.full_iunit
# Spec slicing
self.specslice_position_start = self.spec.index[0]
self.specslice_position_end = self.spec.index[-1]
self.specslider_start = self.spec.index[0]
self.specslider_end = self.spec.index[-1]
self.specstep = (self.spec.index.max() -
self.spec.index.min()) / self.SLIDER_STEPS
self.specspacing = 1
self.timeslice_position_start = self.spec.columns[0]
self.timeslice_position_end = self.spec.columns[-1]
self.timeslider_start = self.spec.columns[0]
self.timeslider_end = self.spec.columns[-1]
self.timestep = 10 #(self.spec.columns.max() - self.spec.columns.min())/self.SLIDER_STEPS
self.timespacing = 1
# Plot defaults to color map
self._color_state = False
self._FREEZE = False
self.draw(name, old, new)
def _norm_unit_changed(self, name, old, new):
self.spec_modified = self.spec_modified.as_norm(
self.NORMUNITS_REV[new])
self.draw(name, old, new)
def _spec_unit_changed(self, name, old, new):
self.spec_modified.specunit = self.SPECUNITS_REV[new]
self.draw(name, old, new)
def _var_unit_changed(self, name, old, new):
self.spec_modified.varunit = self.VARUNITS_REV[new]
self.draw(name, old, new)
def _iunit_changed(self, name, old, new):
self.spec_modified.iunit = new
self.draw(name, old, new)
# Plotting events
# ---------------
def _figwidth_changed(self, name, old, new):
self.draw(name, old, new)
def _figheight_changed(self, name, old, new):
self.draw(name, old, new)
def _colormap_changed(self, name, old, new):
self._color_state = False
self.draw(name, old, new)
def _color_changed(self):
""" Because this sets colorbar, might cause 2 redraws,
so _FREEZE used to prevent this
"""
self._FREEZE = True
self.colorbar = False
self._color_state = True
self._FREEZE = False
self.draw()
def _colorbar_changed(self, name, old, new):
self._color_state = False
self.draw(name, old, new)
def _interactive_changed(self, name, old, new):
self.draw(name, old, new)
# This should be phased out; plots should support colormap, area should handle accordingly
def _kind_changed(self, name, old, new):
self.draw(name, old, new)
def _selectlines_changed(self, name, old, new):
if self.interactive:
self.draw(name, old, new)
# IO Events
# ---------
# THIS SHOULD BE LOAD BUTTON CLICKED!!!!
def _file_name_changed(self):
try:
self.spec = getattr(skspec.data, self.file_name)()
except AttributeError:
pass
@log_message
def save_plot(self):
self.fig_old.savefig(self.save_spec_as + '.png')
@log_message
def save_to_ns(self):
get_ipython().user_ns[self.save_spec_as] = self.spec_modified
@log_message
def load_from_ns(self):
self.spec = get_ipython().user_ns[self.file_name].as_varunit("s")
#def load_from_ns(self, var):
#self.spec = get_ipython().user_ns[var]
# Slicing events
# --------------
def _specslice_position_start_changed(self, name, old, new):
if not self._FREEZE:
self.slice_spectrum(name)
self.draw(name, old, new)
def _specslice_position_end_changed(self, name, old, new):
if not self._FREEZE:
self.slice_spectrum(name)
self.draw(name, old, new)
def _timeslice_position_start_changed(self, name, old, new):
if not self._FREEZE:
self.slice_spectrum(name)
self.draw(name, old, new)
def _timeslice_position_end_changed(self, name, old, new):
if not self._FREEZE:
self.slice_spectrum(name)
self.draw(name, old, new)
def _timespacing_changed(self, name, old, new):
""" Don't let user set less than 1 or more than dataset size"""
# Will have to update when add var/spec slicing
#axis = self.slice_axis
if self.timespacing < 1:
self.timespacing = 1
elif self.timespacing > self.spec_modified.shape[1]:
self.timespacing = self.spec_modified.shape[1]
self.slice_spectrum(name)
self.draw(name, old, new)
def _specspacing_changed(self, name, old, new):
""" Don't let user set less than 1 or more than dataset size"""
# Will have to update when add var/spec slicing
#axis = self.slice_axis
if self.specspacing < 0:
self.specspacing = 0
elif self.specspacing > self.spec_modified.shape[0]:
self.specspacing = self.spec_modified.shape[0]
self.slice_spectrum(name)
self.draw(name, old, new)
# Draw/Slice updates
# ------------------
@log_message
def slice_spectrum(self, name=None):
""" Slice and resample spectra """
self.spec_modified = self.spec.nearby[
self.specslice_position_start:self.specslice_position_end:self.
specspacing, self.timeslice_position_start:self.
timeslice_position_end:self.timespacing]
@log_message
def apply_userf(self, name=None):
import numpy as np
self.spec_modified = self.spec_modified.apply(eval(self.user_f))
self.draw(name)
@log_message
def draw(self, name=None, old=None, new=None):
if name is not None and self.DONT_DRAW.match(name):
return
if self._FREEZE:
return
plot_and_message = ''
# Better way would be a decorator or something that only goes into draw if not autoupdate
if self.autoupdate:
# Generate new figure object
f = plt.figure(figsize=(self.figwidth, self.figheight))
if PLOTPARSER.is_3d(self.kind):
projection = '3d'
else:
projection = None
ax = f.add_subplot(111, projection=projection)
if self._color_state or self.kind not in [
'spec', 'waterfall', 'contour', 'contour3d'
]:
colorkwags = dict(color=self.color)
else:
colorkwags = dict(cmap=self.colormap, cbar=self.colorbar)
self.spec_modified.plot(ax=ax,
fig=f,
kind=self.kind,
norm=self.NORMUNITS_REV[self.norm_unit],
**colorkwags)
f.tight_layout() #Padding around plot
lines = ax.get_lines()
plt.close(f)
#http://mpld3.github.io/modules/API.html
if self.interactive:
import mpld3
if self.selectlines:
from line_plugin import HighlightLines
for idx, col in enumerate(self.spec_modified.columns):
name = 'COLUMN(%s): %s' % (idx, col)
tooltip = mpld3.plugins.LineLabelTooltip(
lines[idx], name)
#voffset=10, hoffset=10, css=css)
mpld3.plugins.connect(f, tooltip)
mpld3.plugins.connect(f, HighlightLines(lines))
plot_and_message += mpld3.fig_to_html(f)
else:
plot_and_message += mpl2html(f)
self.fig_old = f
else:
plot_and_message += html_figure(self.fig_old)
# VALUE IS WHAT GUI LOOKS UP!!!
self.value = plot_and_message
class IPCompleter(Completer):
"""Extension of the completer class with IPython-specific features"""
def _greedy_changed(self, name, old, new):
"""update the splitter and readline delims when greedy is changed"""
if new:
self.splitter.delims = GREEDY_DELIMS
else:
self.splitter.delims = DELIMS
if self.readline:
self.readline.set_completer_delims(self.splitter.delims)
merge_completions = CBool(
True,
config=True,
help="""Whether to merge completion results into a single list
If False, only the completion results from the first non-empty
completer will be returned.
""")
omit__names = Enum(
(0, 1, 2),
default_value=2,
config=True,
help="""Instruct the completer to omit private method names
Specifically, when completing on ``object.<tab>``.
When 2 [default]: all names that start with '_' will be excluded.
When 1: all 'magic' names (``__foo__``) will be excluded.
When 0: nothing will be excluded.
""")
limit_to__all__ = CBool(
default_value=False,
config=True,
help="""Instruct the completer to use __all__ for the completion
Specifically, when completing on ``object.<tab>``.
When True: only those names in obj.__all__ will be included.
When False [default]: the __all__ attribute is ignored
""")
def __init__(self,
shell=None,
namespace=None,
global_namespace=None,
use_readline=True,
config=None,
**kwargs):
"""IPCompleter() -> completer
Return a completer object suitable for use by the readline library
via readline.set_completer().
Inputs:
- shell: a pointer to the ipython shell itself. This is needed
because this completer knows about magic functions, and those can
only be accessed via the ipython instance.
- namespace: an optional dict where completions are performed.
- global_namespace: secondary optional dict for completions, to
handle cases (such as IPython embedded inside functions) where
both Python scopes are visible.
use_readline : bool, optional
If true, use the readline library. This completer can still function
without readline, though in that case callers must provide some extra
information on each call about the current line."""
self.magic_escape = ESC_MAGIC
self.splitter = CompletionSplitter()
# Readline configuration, only used by the rlcompleter method.
if use_readline:
# We store the right version of readline so that later code
import IPython.utils.rlineimpl as readline
self.readline = readline
else:
self.readline = None
# _greedy_changed() depends on splitter and readline being defined:
Completer.__init__(self,
namespace=namespace,
global_namespace=global_namespace,
config=config,
**kwargs)
# List where completion matches will be stored
self.matches = []
self.shell = shell
# Regexp to split filenames with spaces in them
self.space_name_re = re.compile(r'([^\\] )')
# Hold a local ref. to glob.glob for speed
self.glob = glob.glob
# Determine if we are running on 'dumb' terminals, like (X)Emacs
# buffers, to avoid completion problems.
term = os.environ.get('TERM', 'xterm')
self.dumb_terminal = term in ['dumb', 'emacs']
# Special handling of backslashes needed in win32 platforms
if sys.platform == "win32":
self.clean_glob = self._clean_glob_win32
else:
self.clean_glob = self._clean_glob
#regexp to parse docstring for function signature
self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
#use this if positional argument name is also needed
#= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
# All active matcher routines for completion
self.matchers = [
self.python_matches,
self.file_matches,
self.magic_matches,
self.python_func_kw_matches,
self.dict_key_matches,
]
def all_completions(self, text):
"""
Wrapper around the complete method for the benefit of emacs
and pydb.
"""
return self.complete(text)[1]
def _clean_glob(self, text):
return self.glob("%s*" % text)
def _clean_glob_win32(self, text):
return [f.replace("\\", "/") for f in self.glob("%s*" % text)]
def file_matches(self, text):
"""Match filenames, expanding ~USER type strings.
Most of the seemingly convoluted logic in this completer is an
attempt to handle filenames with spaces in them. And yet it's not
quite perfect, because Python's readline doesn't expose all of the
GNU readline details needed for this to be done correctly.
For a filename with a space in it, the printed completions will be
only the parts after what's already been typed (instead of the
full completions, as is normally done). I don't think with the
current (as of Python 2.3) Python readline it's possible to do
better."""
#io.rprint('Completer->file_matches: <%r>' % text) # dbg
# chars that require escaping with backslash - i.e. chars
# that readline treats incorrectly as delimiters, but we
# don't want to treat as delimiters in filename matching
# when escaped with backslash
if text.startswith('!'):
text = text[1:]
text_prefix = '!'
else:
text_prefix = ''
text_until_cursor = self.text_until_cursor
# track strings with open quotes
open_quotes = has_open_quotes(text_until_cursor)
if '(' in text_until_cursor or '[' in text_until_cursor:
lsplit = text
else:
try:
# arg_split ~ shlex.split, but with unicode bugs fixed by us
lsplit = arg_split(text_until_cursor)[-1]
except ValueError:
# typically an unmatched ", or backslash without escaped char.
if open_quotes:
lsplit = text_until_cursor.split(open_quotes)[-1]
else:
return []
except IndexError:
# tab pressed on empty line
lsplit = ""
if not open_quotes and lsplit != protect_filename(lsplit):
# if protectables are found, do matching on the whole escaped name
has_protectables = True
text0, text = text, lsplit
else:
has_protectables = False
text = os.path.expanduser(text)
if text == "":
return [text_prefix + protect_filename(f) for f in self.glob("*")]
# Compute the matches from the filesystem
m0 = self.clean_glob(text.replace('\\', ''))
if has_protectables:
# If we had protectables, we need to revert our changes to the
# beginning of filename so that we don't double-write the part
# of the filename we have so far
len_lsplit = len(lsplit)
matches = [
text_prefix + text0 + protect_filename(f[len_lsplit:])
for f in m0
]
else:
if open_quotes:
# if we have a string with an open quote, we don't need to
# protect the names at all (and we _shouldn't_, as it
# would cause bugs when the filesystem call is made).
matches = m0
else:
matches = [text_prefix + protect_filename(f) for f in m0]
#io.rprint('mm', matches) # dbg
# Mark directories in input list by appending '/' to their names.
matches = [x + '/' if os.path.isdir(x) else x for x in matches]
return matches
def magic_matches(self, text):
"""Match magics"""
#print 'Completer->magic_matches:',text,'lb',self.text_until_cursor # dbg
# Get all shell magics now rather than statically, so magics loaded at
# runtime show up too.
lsm = self.shell.magics_manager.lsmagic()
line_magics = lsm['line']
cell_magics = lsm['cell']
pre = self.magic_escape
pre2 = pre + pre
# Completion logic:
# - user gives %%: only do cell magics
# - user gives %: do both line and cell magics
# - no prefix: do both
# In other words, line magics are skipped if the user gives %% explicitly
bare_text = text.lstrip(pre)
comp = [pre2 + m for m in cell_magics if m.startswith(bare_text)]
if not text.startswith(pre2):
comp += [pre + m for m in line_magics if m.startswith(bare_text)]
return comp
def python_matches(self, text):
"""Match attributes or global python names"""
#io.rprint('Completer->python_matches, txt=%r' % text) # dbg
if "." in text:
try:
matches = self.attr_matches(text)
if text.endswith('.') and self.omit__names:
if self.omit__names == 1:
# true if txt is _not_ a __ name, false otherwise:
no__name = (
lambda txt: re.match(r'.*\.__.*?__', txt) is None)
else:
# true if txt is _not_ a _ name, false otherwise:
no__name = (lambda txt: re.match(
r'\._.*?', txt[txt.rindex('.'):]) is None)
matches = filter(no__name, matches)
except NameError:
# catches <undefined attributes>.<tab>
matches = []
else:
matches = self.global_matches(text)
return matches
def _default_arguments_from_docstring(self, doc):
"""Parse the first line of docstring for call signature.
Docstring should be of the form 'min(iterable[, key=func])\n'.
It can also parse cython docstring of the form
'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
"""
if doc is None:
return []
#care only the firstline
line = doc.lstrip().splitlines()[0]
#p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
#'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
sig = self.docstring_sig_re.search(line)
if sig is None:
return []
# iterable[, key=func]' -> ['iterable[' ,' key=func]']
sig = sig.groups()[0].split(',')
ret = []
for s in sig:
#re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
ret += self.docstring_kwd_re.findall(s)
return ret
def _default_arguments(self, obj):
"""Return the list of default arguments of obj if it is callable,
or empty list otherwise."""
call_obj = obj
ret = []
if inspect.isbuiltin(obj):
pass
elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
if inspect.isclass(obj):
#for cython embededsignature=True the constructor docstring
#belongs to the object itself not __init__
ret += self._default_arguments_from_docstring(
getattr(obj, '__doc__', ''))
# for classes, check for __init__,__new__
call_obj = (getattr(obj, '__init__', None)
or getattr(obj, '__new__', None))
# for all others, check if they are __call__able
elif hasattr(obj, '__call__'):
call_obj = obj.__call__
ret += self._default_arguments_from_docstring(
getattr(call_obj, '__doc__', ''))
try:
args, _, _1, defaults = inspect.getargspec(call_obj)
if defaults:
ret += args[-len(defaults):]
except TypeError:
pass
return list(set(ret))
def python_func_kw_matches(self, text):
"""Match named parameters (kwargs) of the last open function"""
if "." in text: # a parameter cannot be dotted
return []
try:
regexp = self.__funcParamsRegex
except AttributeError:
regexp = self.__funcParamsRegex = re.compile(
r'''
'.*?(?<!\\)' | # single quoted strings or
".*?(?<!\\)" | # double quoted strings or
\w+ | # identifier
\S # other characters
''', re.VERBOSE | re.DOTALL)
# 1. find the nearest identifier that comes before an unclosed
# parenthesis before the cursor
# e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
tokens = regexp.findall(self.text_until_cursor)
tokens.reverse()
iterTokens = iter(tokens)
openPar = 0
for token in iterTokens:
if token == ')':
openPar -= 1
elif token == '(':
openPar += 1
if openPar > 0:
# found the last unclosed parenthesis
break
else:
return []
# 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
ids = []
isId = re.compile(r'\w+$').match
while True:
try:
ids.append(next(iterTokens))
if not isId(ids[-1]):
ids.pop()
break
if not next(iterTokens) == '.':
break
except StopIteration:
break
# lookup the candidate callable matches either using global_matches
# or attr_matches for dotted names
if len(ids) == 1:
callableMatches = self.global_matches(ids[0])
else:
callableMatches = self.attr_matches('.'.join(ids[::-1]))
argMatches = []
for callableMatch in callableMatches:
try:
namedArgs = self._default_arguments(
eval(callableMatch, self.namespace))
except:
continue
for namedArg in namedArgs:
if namedArg.startswith(text):
argMatches.append("%s=" % namedArg)
return argMatches
def dict_key_matches(self, text):
"Match string keys in a dictionary, after e.g. 'foo[' "
def get_keys(obj):
# Only allow completion for known in-memory dict-like types
if isinstance(obj, dict) or\
_safe_isinstance(obj, 'pandas', 'DataFrame'):
try:
return list(obj.keys())
except Exception:
return []
elif _safe_isinstance(obj, 'numpy', 'ndarray'):
return obj.dtype.names or []
return []
try:
regexps = self.__dict_key_regexps
except AttributeError:
dict_key_re_fmt = r'''(?x)
( # match dict-referring expression wrt greedy setting
%s
)
\[ # open bracket
\s* # and optional whitespace
([uUbB]? # string prefix (r not handled)
(?: # unclosed string
'(?:[^']|(?<!\\)\\')*
|
"(?:[^"]|(?<!\\)\\")*
)
)?
$
'''
regexps = self.__dict_key_regexps = {
False:
re.compile(dict_key_re_fmt % '''
# identifiers separated by .
(?!\d)\w+
(?:\.(?!\d)\w+)*
'''),
True:
re.compile(dict_key_re_fmt % '''
.+
''')
}
match = regexps[self.greedy].search(self.text_until_cursor)
if match is None:
return []
expr, prefix = match.groups()
try:
obj = eval(expr, self.namespace)
except Exception:
try:
obj = eval(expr, self.global_namespace)
except Exception:
return []
keys = get_keys(obj)
if not keys:
return keys
closing_quote, token_offset, matches = match_dict_keys(keys, prefix)
if not matches:
return matches
# get the cursor position of
# - the text being completed
# - the start of the key text
# - the start of the completion
text_start = len(self.text_until_cursor) - len(text)
if prefix:
key_start = match.start(2)
completion_start = key_start + token_offset
else:
key_start = completion_start = match.end()
# grab the leading prefix, to make sure all completions start with `text`
if text_start > key_start:
leading = ''
else:
leading = text[text_start:completion_start]
# the index of the `[` character
bracket_idx = match.end(1)
# append closing quote and bracket as appropriate
# this is *not* appropriate if the opening quote or bracket is outside
# the text given to this method
suf = ''
continuation = self.line_buffer[len(self.text_until_cursor):]
if key_start > text_start and closing_quote:
# quotes were opened inside text, maybe close them
if continuation.startswith(closing_quote):
continuation = continuation[len(closing_quote):]
else:
suf += closing_quote
if bracket_idx > text_start:
# brackets were opened inside text, maybe close them
if not continuation.startswith(']'):
suf += ']'
return [leading + k + suf for k in matches]
def latex_matches(self, text):
u"""Match Latex syntax for unicode characters.
This does both \\alp -> \\alpha and \\alpha -> α
Used on Python 3 only.
"""
slashpos = text.rfind('\\')
if slashpos > -1:
s = text[slashpos:]
if s in latex_symbols:
# Try to complete a full latex symbol to unicode
# \\alpha -> α
return s, [latex_symbols[s]]
else:
# If a user has partially typed a latex symbol, give them
# a full list of options \al -> [\aleph, \alpha]
matches = [k for k in latex_symbols if k.startswith(s)]
return s, matches
return u'', []
def dispatch_custom_completer(self, text):
#io.rprint("Custom! '%s' %s" % (text, self.custom_completers)) # dbg
line = self.line_buffer
if not line.strip():
return None
# Create a little structure to pass all the relevant information about
# the current completion to any custom completer.
event = Bunch()
event.line = line
event.symbol = text
cmd = line.split(None, 1)[0]
event.command = cmd
event.text_until_cursor = self.text_until_cursor
#print "\ncustom:{%s]\n" % event # dbg
# for foo etc, try also to find completer for %foo
if not cmd.startswith(self.magic_escape):
try_magic = self.custom_completers.s_matches(self.magic_escape +
cmd)
else:
try_magic = []
for c in itertools.chain(
self.custom_completers.s_matches(cmd), try_magic,
self.custom_completers.flat_matches(self.text_until_cursor)):
#print "try",c # dbg
try:
res = c(event)
if res:
# first, try case sensitive match
withcase = [r for r in res if r.startswith(text)]
if withcase:
return withcase
# if none, then case insensitive ones are ok too
text_low = text.lower()
return [r for r in res if r.lower().startswith(text_low)]
except TryNext:
pass
return None
def complete(self, text=None, line_buffer=None, cursor_pos=None):
"""Find completions for the given text and line context.
Note that both the text and the line_buffer are optional, but at least
one of them must be given.
Parameters
----------
text : string, optional
Text to perform the completion on. If not given, the line buffer
is split using the instance's CompletionSplitter object.
line_buffer : string, optional
If not given, the completer attempts to obtain the current line
buffer via readline. This keyword allows clients which are
requesting for text completions in non-readline contexts to inform
the completer of the entire text.
cursor_pos : int, optional
Index of the cursor in the full line buffer. Should be provided by
remote frontends where kernel has no access to frontend state.
Returns
-------
text : str
Text that was actually used in the completion.
matches : list
A list of completion matches.
"""
# io.rprint('\nCOMP1 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg
# if the cursor position isn't given, the only sane assumption we can
# make is that it's at the end of the line (the common case)
if cursor_pos is None:
cursor_pos = len(line_buffer) if text is None else len(text)
if PY3:
latex_text = text if not line_buffer else line_buffer[:cursor_pos]
latex_text, latex_matches = self.latex_matches(latex_text)
if latex_matches:
return latex_text, latex_matches
# if text is either None or an empty string, rely on the line buffer
if not text:
text = self.splitter.split_line(line_buffer, cursor_pos)
# If no line buffer is given, assume the input text is all there was
if line_buffer is None:
line_buffer = text
self.line_buffer = line_buffer
self.text_until_cursor = self.line_buffer[:cursor_pos]
# io.rprint('COMP2 %r %r %r' % (text, line_buffer, cursor_pos)) # dbg
# Start with a clean slate of completions
self.matches[:] = []
custom_res = self.dispatch_custom_completer(text)
if custom_res is not None:
# did custom completers produce something?
self.matches = custom_res
else:
# Extend the list of completions with the results of each
# matcher, so we return results to the user from all
# namespaces.
if self.merge_completions:
self.matches = []
for matcher in self.matchers:
try:
self.matches.extend(matcher(text))
except:
# Show the ugly traceback if the matcher causes an
# exception, but do NOT crash the kernel!
sys.excepthook(*sys.exc_info())
else:
for matcher in self.matchers:
self.matches = matcher(text)
if self.matches:
break
# FIXME: we should extend our api to return a dict with completions for
# different types of objects. The rlcomplete() method could then
# simply collapse the dict into a list for readline, but we'd have
# richer completion semantics in other evironments.
# use penalize_magics_key to put magics after variables with same name
self.matches = sorted(set(self.matches), key=penalize_magics_key)
#io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
return text, self.matches
def rlcomplete(self, text, state):
"""Return the state-th possible completion for 'text'.
This is called successively with state == 0, 1, 2, ... until it
returns None. The completion should begin with 'text'.
Parameters
----------
text : string
Text to perform the completion on.
state : int
Counter used by readline.
"""
if state == 0:
self.line_buffer = line_buffer = self.readline.get_line_buffer()
cursor_pos = self.readline.get_endidx()
#io.rprint("\nRLCOMPLETE: %r %r %r" %
# (text, line_buffer, cursor_pos) ) # dbg
# if there is only a tab on a line with only whitespace, instead of
# the mostly useless 'do you want to see all million completions'
# message, just do the right thing and give the user his tab!
# Incidentally, this enables pasting of tabbed text from an editor
# (as long as autoindent is off).
# It should be noted that at least pyreadline still shows file
# completions - is there a way around it?
# don't apply this on 'dumb' terminals, such as emacs buffers, so
# we don't interfere with their own tab-completion mechanism.
if not (self.dumb_terminal or line_buffer.strip()):
self.readline.insert_text('\t')
sys.stdout.flush()
return None
# Note: debugging exceptions that may occur in completion is very
# tricky, because readline unconditionally silences them. So if
# during development you suspect a bug in the completion code, turn
# this flag on temporarily by uncommenting the second form (don't
# flip the value in the first line, as the '# dbg' marker can be
# automatically detected and is used elsewhere).
DEBUG = False
#DEBUG = True # dbg
if DEBUG:
try:
self.complete(text, line_buffer, cursor_pos)
except:
import traceback
traceback.print_exc()
else:
# The normal production version is here
# This method computes the self.matches array
self.complete(text, line_buffer, cursor_pos)
try:
return self.matches[state]
except IndexError:
return None
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
default_config_file_name = 'ipython_notebook_config.py'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = IPythonConsoleApp.classes + [
MappingKernelManager, NotebookManager
]
flags = Dict(flags)
aliases = Dict(aliases)
kernel_argv = List(Unicode)
log_level = Enum(
(0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'),
default_value=logging.INFO,
config=True,
help="Set the log level by value or name.")
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('')
# Network related information.
ip = Unicode(LOCALHOST,
config=True,
help="The IP address the notebook server will listen on.")
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888,
config=True,
help="The port the notebook server will listen on.")
port_retries = Integer(
50,
config=True,
help=
"The number of additional ports to try if the specified port is not available."
)
certfile = Unicode(
u'',
config=True,
help="""The full path to an SSL/TLS certificate file.""")
keyfile = Unicode(
u'',
config=True,
help="""The full path to a private key file for usage with SSL/TLS.""")
password = Unicode(u'',
config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
""")
open_browser = Bool(True,
config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode(u'',
config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
read_only = Bool(False,
config=True,
help="Whether to prevent editing/execution of notebooks.")
webapp_settings = Dict(
config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
enable_mathjax = Bool(
True,
config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
""")
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
base_project_url = Unicode('/',
config=True,
help='''The base URL for the notebook server''')
base_kernel_url = Unicode('/',
config=True,
help='''The base URL for the kernel server''')
websocket_host = Unicode("",
config=True,
help="""The hostname for the websocket server.""")
mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""")
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_path = self.webapp_settings.get(
"static_path", os.path.join(os.path.dirname(__file__), "static"))
static_url_prefix = self.webapp_settings.get("static_url_prefix",
"/static/")
if os.path.exists(os.path.join(static_path, 'mathjax', "MathJax.js")):
self.log.info("Using local MathJax")
return static_url_prefix + u"mathjax/MathJax.js"
else:
if self.certfile:
# HTTPS: load from Rackspace CDN, because SSL certificate requires it
base = u"https://c328740.ssl.cf1.rackcdn.com"
else:
base = u"http://cdn.mathjax.org"
url = base + u"/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if argv is None:
argv = sys.argv[1:]
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--KernelApp.parent_appname='%s'" % self.name)
if self.extra_args:
f = os.path.abspath(self.extra_args[0])
if os.path.isdir(f):
nbdir = f
else:
self.file_to_run = f
nbdir = os.path.dirname(f)
self.config.NotebookManager.notebook_dir = nbdir
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_manager = MappingKernelManager(
config=self.config,
log=self.log,
kernel_argv=self.kernel_argv,
connection_dir=self.profile_dir.security_dir,
)
self.notebook_manager = NotebookManager(config=self.config,
log=self.log)
self.log.info("Serving notebooks from %s",
self.notebook_manager.notebook_dir)
self.notebook_manager.list_notebooks()
self.cluster_manager = ClusterManager(config=self.config, log=self.log)
self.cluster_manager.update_profiles()
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(self, self.kernel_manager,
self.notebook_manager,
self.cluster_manager, self.log,
self.base_project_url,
self.webapp_settings)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app,
ssl_options=ssl_options)
if not self.ip:
warning = "WARNING: The notebook server is listening on all IP addresses"
if ssl_options is None:
self.log.critical(warning + " and not using encryption. This"
"is not recommended.")
if not self.password and not self.read_only:
self.log.critical(
warning + "and not using authentication."
"This is highly insecure and not recommended.")
success = None
for port in random_ports(self.port, self.port_retries + 1):
try:
self.http_server.listen(port, self.ip)
except socket.error, e:
if e.errno != errno.EADDRINUSE:
raise
self.log.info(
'The port %i is already in use, trying another random port.'
% port)
else:
self.port = port
success = True
break
if not success:
self.log.critical(
'ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
class RContent(Reference):
status = Enum((u'ok', u'error'))
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
default_config_file_name='ipython_notebook_config.py'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager]
flags = Dict(flags)
aliases = Dict(aliases)
kernel_argv = List(Unicode)
log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
default_value=logging.INFO,
config=True,
help="Set the log level by value or name.")
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# file to be opened in the notebook server
file_to_run = Unicode('')
# Network related information.
ip = Unicode(LOCALHOST, config=True,
help="The IP address the notebook server will listen on."
)
def _ip_changed(self, name, old, new):
if new == '*': self.ip = ''
port = Integer(8888, config=True,
help="The port the notebook server will listen on."
)
port_retries = Integer(50, config=True,
help="The number of additional ports to try if the specified port is not available."
)
certfile = Unicode('', config=True,
help="""The full path to an SSL/TLS certificate file."""
)
keyfile = Unicode('', config=True,
help="""The full path to a private key file for usage with SSL/TLS."""
)
password = Unicode('', config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
"""
)
open_browser = Bool(True, config=True,
help="""Whether to open in a browser after starting.
The specific browser used is platform dependent and
determined by the python standard library `webbrowser`
module, unless it is overridden using the --browser
(NotebookApp.browser) configuration option.
""")
browser = Unicode('', config=True,
help="""Specify what command to use to invoke a web
browser when opening the notebook. If not specified, the
default browser will be determined by the `webbrowser`
standard library module, which allows setting of the
BROWSER environment variable to override it.
""")
read_only = Bool(False, config=True,
help="Whether to prevent editing/execution of notebooks."
)
webapp_settings = Dict(config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
enable_mathjax = Bool(True, config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
"""
)
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = ''
base_project_url = Unicode('/', config=True,
help='''The base URL for the notebook server''')
base_kernel_url = Unicode('/', config=True,
help='''The base URL for the kernel server''')
websocket_host = Unicode("", config=True,
help="""The hostname for the websocket server."""
)
mathjax_url = Unicode("", config=True,
help="""The url for MathJax.js."""
)
def _mathjax_url_default(self):
if not self.enable_mathjax:
return ''
static_path = self.webapp_settings.get("static_path", os.path.join(os.path.dirname(__file__), "static"))
static_url_prefix = self.webapp_settings.get("static_url_prefix",
"/static/")
if os.path.exists(os.path.join(static_path, 'mathjax', "MathJax.js")):
self.log.info("Using local MathJax")
return static_url_prefix+"mathjax/MathJax.js"
else:
if self.certfile:
# HTTPS: load from Rackspace CDN, because SSL certificate requires it
base = "https://c328740.ssl.cf1.rackcdn.com"
else:
base = "http://cdn.mathjax.org"
url = base + "/mathjax/latest/MathJax.js"
self.log.info("Using MathJax from CDN: %s", url)
return url
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = ''
else:
self.log.info("Using MathJax: %s", new)
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if argv is None:
argv = sys.argv[1:]
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
if self.extra_args:
f = os.path.abspath(self.extra_args[0])
if os.path.isdir(f):
nbdir = f
else:
self.file_to_run = f
nbdir = os.path.dirname(f)
self.config.NotebookManager.notebook_dir = nbdir
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
self.kernel_manager = MappingKernelManager(
config=self.config, log=self.log, kernel_argv=self.kernel_argv,
connection_dir = self.profile_dir.security_dir,
)
self.notebook_manager = NotebookManager(config=self.config, log=self.log)
self.log.info("Serving notebooks from %s", self.notebook_manager.notebook_dir)
self.notebook_manager.list_notebooks()
self.cluster_manager = ClusterManager(config=self.config, log=self.log)
self.cluster_manager.update_profiles()
def init_logging(self):
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(
self, self.kernel_manager, self.notebook_manager,
self.cluster_manager, self.log,
self.base_project_url, self.webapp_settings
)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
if ssl_options is None and not self.ip and not (self.read_only and not self.password):
self.log.critical('WARNING: the notebook server is listening on all IP addresses '
'but not using any encryption or authentication. This is highly '
'insecure and not recommended.')
success = None
for port in random_ports(self.port, self.port_retries+1):
try:
self.http_server.listen(port, self.ip)
except socket.error as e:
if e.errno != errno.EADDRINUSE:
raise
self.log.info('The port %i is already in use, trying another random port.' % port)
else:
self.port = port
success = True
break
if not success:
self.log.critical('ERROR: the notebook server could not be started because '
'no available port could be found.')
self.exit(1)
def init_signal(self):
# FIXME: remove this check when pyzmq dependency is >= 2.1.11
# safely extract zmq version info:
try:
zmq_v = zmq.pyzmq_version_info()
except AttributeError:
zmq_v = [ int(n) for n in re.findall(r'\d+', zmq.__version__) ]
if 'dev' in zmq.__version__:
zmq_v.append(999)
zmq_v = tuple(zmq_v)
if zmq_v >= (2,1,9) and not sys.platform.startswith('win'):
# This won't work with 2.1.7 and
# 2.1.9-10 will log ugly 'Interrupted system call' messages,
# but it will work
signal.signal(signal.SIGINT, self._handle_sigint)
signal.signal(signal.SIGTERM, self._signal_stop)
def _handle_sigint(self, sig, frame):
"""SIGINT handler spawns confirmation dialog"""
# register more forceful signal handler for ^C^C case
signal.signal(signal.SIGINT, self._signal_stop)
# request confirmation dialog in bg thread, to avoid
# blocking the App
thread = threading.Thread(target=self._confirm_exit)
thread.daemon = True
thread.start()
def _restore_sigint_handler(self):
"""callback for restoring original SIGINT handler"""
signal.signal(signal.SIGINT, self._handle_sigint)
def _confirm_exit(self):
"""confirm shutdown on ^C
A second ^C, or answering 'y' within 5s will cause shutdown,
otherwise original SIGINT handler will be restored.
This doesn't work on Windows.
"""
# FIXME: remove this delay when pyzmq dependency is >= 2.1.11
time.sleep(0.1)
sys.stdout.write("Shutdown Notebook Server (y/[n])? ")
sys.stdout.flush()
r,w,x = select.select([sys.stdin], [], [], 5)
if r:
line = sys.stdin.readline()
if line.lower().startswith('y'):
self.log.critical("Shutdown confirmed")
ioloop.IOLoop.instance().stop()
return
else:
print("No answer for 5s:", end=' ')
print("resuming operation...")
# no answer, or answer is no:
# set it back to original SIGINT handler
# use IOLoop.add_callback because signal.signal must be called
# from main thread
ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
def _signal_stop(self, sig, frame):
self.log.critical("received signal %s, stopping", sig)
ioloop.IOLoop.instance().stop()
@catch_config_error
def initialize(self, argv=None):
self.init_logging()
super(NotebookApp, self).initialize(argv)
self.init_configurables()
self.init_webapp()
self.init_signal()
def cleanup_kernels(self):
"""shutdown all kernels
The kernels will shutdown themselves when this process no longer exists,
but explicit shutdown allows the KernelManagers to cleanup the connection files.
"""
self.log.info('Shutting down kernels')
km = self.kernel_manager
# copy list, since shutdown_kernel deletes keys
for kid in list(km.kernel_ids):
km.shutdown_kernel(kid)
def start(self):
ip = self.ip if self.ip else '[all ip addresses on your system]'
proto = 'https' if self.certfile else 'http'
info = self.log.info
info("The IPython Notebook is running at: %s://%s:%i%s" %
(proto, ip, self.port,self.base_project_url) )
info("Use Control-C to stop this server and shut down all kernels.")
if self.open_browser or self.file_to_run:
ip = self.ip or '127.0.0.1'
try:
browser = webbrowser.get(self.browser or None)
except webbrowser.Error as e:
self.log.warn('No web browser found: %s.' % e)
browser = None
if self.file_to_run:
filename, _ = os.path.splitext(os.path.basename(self.file_to_run))
for nb in self.notebook_manager.list_notebooks():
if filename == nb['name']:
url = nb['notebook_id']
break
else:
url = ''
else:
url = ''
if browser:
b = lambda : browser.open("%s://%s:%i%s%s" % (proto, ip,
self.port, self.base_project_url, url), new=2)
threading.Thread(target=b).start()
try:
ioloop.IOLoop.instance().start()
except KeyboardInterrupt:
info("Interrupted...")
finally:
self.cleanup_kernels()
class Stream(Reference):
name = Enum((u'stdout', u'stderr'))
data = Unicode()
class TaskScheduler(SessionFactory):
"""Python TaskScheduler object.
This is the simplest object that supports msg_id based
DAG dependencies. *Only* task msg_ids are checked, not
msg_ids of jobs submitted via the MUX queue.
"""
hwm = Integer(1,
config=True,
help="""specify the High Water Mark (HWM) for the downstream
socket in the Task scheduler. This is the maximum number
of allowed outstanding tasks on each engine.
The default (1) means that only one task can be outstanding on each
engine. Setting TaskScheduler.hwm=0 means there is no limit, and the
engines continue to be assigned tasks while they are working,
effectively hiding network latency behind computation, but can result
in an imbalance of work when submitting many heterogenous tasks all at
once. Any positive value greater than one is a compromise between the
two.
""")
scheme_name = Enum(
('leastload', 'pure', 'lru', 'plainrandom', 'weighted', 'twobin'),
'leastload',
config=True,
allow_none=False,
help="""select the task scheduler scheme [default: Python LRU]
Options are: 'pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'"""
)
def _scheme_name_changed(self, old, new):
self.log.debug("Using scheme %r" % new)
self.scheme = globals()[new]
# input arguments:
scheme = Instance(FunctionType) # function for determining the destination
def _scheme_default(self):
return leastload
client_stream = Instance(zmqstream.ZMQStream) # client-facing stream
engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream
notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream
mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream
query_stream = Instance(zmqstream.ZMQStream) # hub-facing DEALER stream
# internals:
graph = Dict() # dict by msg_id of [ msg_ids that depend on key ]
retries = Dict() # dict by msg_id of retries remaining (non-neg ints)
# waiting = List() # list of msg_ids ready to run, but haven't due to HWM
depending = Dict() # dict by msg_id of Jobs
pending = Dict() # dict by engine_uuid of submitted tasks
completed = Dict() # dict by engine_uuid of completed tasks
failed = Dict() # dict by engine_uuid of failed tasks
destinations = Dict(
) # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed)
clients = Dict() # dict by msg_id for who submitted the task
targets = List() # list of target IDENTs
loads = List() # list of engine loads
# full = Set() # set of IDENTs that have HWM outstanding tasks
all_completed = Set() # set of all completed tasks
all_failed = Set() # set of all failed tasks
all_done = Set() # set of all finished tasks=union(completed,failed)
all_ids = Set() # set of all submitted task IDs
auditor = Instance('zmq.eventloop.ioloop.PeriodicCallback')
ident = CBytes() # ZMQ identity. This should just be self.session.session
# but ensure Bytes
def _ident_default(self):
return self.session.bsession
def start(self):
self.query_stream.on_recv(self.dispatch_query_reply)
self.session.send(self.query_stream, "connection_request", {})
self.engine_stream.on_recv(self.dispatch_result, copy=False)
self.client_stream.on_recv(self.dispatch_submission, copy=False)
self._notification_handlers = dict(
registration_notification=self._register_engine,
unregistration_notification=self._unregister_engine)
self.notifier_stream.on_recv(self.dispatch_notification)
self.auditor = ioloop.PeriodicCallback(self.audit_timeouts, 2e3,
self.loop) # 1 Hz
self.auditor.start()
self.log.info("Scheduler started [%s]" % self.scheme_name)
def resume_receiving(self):
"""Resume accepting jobs."""
self.client_stream.on_recv(self.dispatch_submission, copy=False)
def stop_receiving(self):
"""Stop accepting jobs while there are no engines.
Leave them in the ZMQ queue."""
self.client_stream.on_recv(None)
#-----------------------------------------------------------------------
# [Un]Registration Handling
#-----------------------------------------------------------------------
def dispatch_query_reply(self, msg):
"""handle reply to our initial connection request"""
try:
idents, msg = self.session.feed_identities(msg)
except ValueError:
self.log.warn("task::Invalid Message: %r", msg)
return
try:
msg = self.session.unserialize(msg)
except ValueError:
self.log.warn("task::Unauthorized message from: %r" % idents)
return
content = msg['content']
for uuid in content.get('engines', {}).values():
self._register_engine(cast_bytes(uuid))
@util.log_errors
def dispatch_notification(self, msg):
"""dispatch register/unregister events."""
try:
idents, msg = self.session.feed_identities(msg)
except ValueError:
self.log.warn("task::Invalid Message: %r", msg)
return
try:
msg = self.session.unserialize(msg)
except ValueError:
self.log.warn("task::Unauthorized message from: %r" % idents)
return
msg_type = msg['header']['msg_type']
handler = self._notification_handlers.get(msg_type, None)
if handler is None:
self.log.error("Unhandled message type: %r" % msg_type)
else:
try:
handler(cast_bytes(msg['content']['uuid']))
except Exception:
self.log.error("task::Invalid notification msg: %r",
msg,
exc_info=True)
def _register_engine(self, uid):
"""New engine with ident `uid` became available."""
# head of the line:
self.targets.insert(0, uid)
self.loads.insert(0, 0)
# initialize sets
self.completed[uid] = set()
self.failed[uid] = set()
self.pending[uid] = {}
# rescan the graph:
self.update_graph(None)
def _unregister_engine(self, uid):
"""Existing engine with ident `uid` became unavailable."""
if len(self.targets) == 1:
# this was our only engine
pass
# handle any potentially finished tasks:
self.engine_stream.flush()
# don't pop destinations, because they might be used later
# map(self.destinations.pop, self.completed.pop(uid))
# map(self.destinations.pop, self.failed.pop(uid))
# prevent this engine from receiving work
idx = self.targets.index(uid)
self.targets.pop(idx)
self.loads.pop(idx)
# wait 5 seconds before cleaning up pending jobs, since the results might
# still be incoming
if self.pending[uid]:
dc = ioloop.DelayedCallback(
lambda: self.handle_stranded_tasks(uid), 5000, self.loop)
dc.start()
else:
self.completed.pop(uid)
self.failed.pop(uid)
def handle_stranded_tasks(self, engine):
"""Deal with jobs resident in an engine that died."""
lost = self.pending[engine]
for msg_id in lost.keys():
if msg_id not in self.pending[engine]:
# prevent double-handling of messages
continue
raw_msg = lost[msg_id].raw_msg
idents, msg = self.session.feed_identities(raw_msg, copy=False)
parent = self.session.unpack(msg[1].bytes)
idents = [engine, idents[0]]
# build fake error reply
try:
raise error.EngineError(
"Engine %r died while running task %r" % (engine, msg_id))
except:
content = error.wrap_exception()
# build fake metadata
md = dict(
status=u'error',
engine=engine,
date=datetime.now(),
)
msg = self.session.msg('apply_reply',
content,
parent=parent,
metadata=md)
raw_reply = map(zmq.Message,
self.session.serialize(msg, ident=idents))
# and dispatch it
self.dispatch_result(raw_reply)
# finally scrub completed/failed lists
self.completed.pop(engine)
self.failed.pop(engine)
#-----------------------------------------------------------------------
# Job Submission
#-----------------------------------------------------------------------
@util.log_errors
def dispatch_submission(self, raw_msg):
"""Dispatch job submission to appropriate handlers."""
# ensure targets up to date:
self.notifier_stream.flush()
try:
idents, msg = self.session.feed_identities(raw_msg, copy=False)
msg = self.session.unserialize(msg, content=False, copy=False)
except Exception:
self.log.error("task::Invaid task msg: %r" % raw_msg,
exc_info=True)
return
# send to monitor
self.mon_stream.send_multipart([b'intask'] + raw_msg, copy=False)
header = msg['header']
md = msg['metadata']
msg_id = header['msg_id']
self.all_ids.add(msg_id)
# get targets as a set of bytes objects
# from a list of unicode objects
targets = md.get('targets', [])
targets = map(cast_bytes, targets)
targets = set(targets)
retries = md.get('retries', 0)
self.retries[msg_id] = retries
# time dependencies
after = md.get('after', None)
if after:
after = Dependency(after)
if after.all:
if after.success:
after = Dependency(
after.difference(self.all_completed),
success=after.success,
failure=after.failure,
all=after.all,
)
if after.failure:
after = Dependency(
after.difference(self.all_failed),
success=after.success,
failure=after.failure,
all=after.all,
)
if after.check(self.all_completed, self.all_failed):
# recast as empty set, if `after` already met,
# to prevent unnecessary set comparisons
after = MET
else:
after = MET
# location dependencies
follow = Dependency(md.get('follow', []))
# turn timeouts into datetime objects:
timeout = md.get('timeout', None)
if timeout:
# cast to float, because jsonlib returns floats as decimal.Decimal,
# which timedelta does not accept
timeout = datetime.now() + timedelta(0, float(timeout), 0)
job = Job(
msg_id=msg_id,
raw_msg=raw_msg,
idents=idents,
msg=msg,
header=header,
targets=targets,
after=after,
follow=follow,
timeout=timeout,
metadata=md,
)
# validate and reduce dependencies:
for dep in after, follow:
if not dep: # empty dependency
continue
# check valid:
if msg_id in dep or dep.difference(self.all_ids):
self.depending[msg_id] = job
return self.fail_unreachable(msg_id, error.InvalidDependency)
# check if unreachable:
if dep.unreachable(self.all_completed, self.all_failed):
self.depending[msg_id] = job
return self.fail_unreachable(msg_id)
if after.check(self.all_completed, self.all_failed):
# time deps already met, try to run
if not self.maybe_run(job):
# can't run yet
if msg_id not in self.all_failed:
# could have failed as unreachable
self.save_unmet(job)
else:
self.save_unmet(job)
def audit_timeouts(self):
"""Audit all waiting tasks for expired timeouts."""
now = datetime.now()
for msg_id in self.depending.keys():
# must recheck, in case one failure cascaded to another:
if msg_id in self.depending:
job = self.depending[msg_id]
if job.timeout and job.timeout < now:
self.fail_unreachable(msg_id, error.TaskTimeout)
def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
"""a task has become unreachable, send a reply with an ImpossibleDependency
error."""
if msg_id not in self.depending:
self.log.error("msg %r already failed!", msg_id)
return
job = self.depending.pop(msg_id)
for mid in job.dependents:
if mid in self.graph:
self.graph[mid].remove(msg_id)
try:
raise why()
except:
content = error.wrap_exception()
self.all_done.add(msg_id)
self.all_failed.add(msg_id)
msg = self.session.send(self.client_stream,
'apply_reply',
content,
parent=job.header,
ident=job.idents)
self.session.send(self.mon_stream,
msg,
ident=[b'outtask'] + job.idents)
self.update_graph(msg_id, success=False)
def maybe_run(self, job):
"""check location dependencies, and run if they are met."""
msg_id = job.msg_id
self.log.debug("Attempting to assign task %s", msg_id)
if not self.targets:
# no engines, definitely can't run
return False
if job.follow or job.targets or job.blacklist or self.hwm:
# we need a can_run filter
def can_run(idx):
# check hwm
if self.hwm and self.loads[idx] == self.hwm:
return False
target = self.targets[idx]
# check blacklist
if target in job.blacklist:
return False
# check targets
if job.targets and target not in job.targets:
return False
# check follow
return job.follow.check(self.completed[target],
self.failed[target])
indices = filter(can_run, range(len(self.targets)))
if not indices:
# couldn't run
if job.follow.all:
# check follow for impossibility
dests = set()
relevant = set()
if job.follow.success:
relevant = self.all_completed
if job.follow.failure:
relevant = relevant.union(self.all_failed)
for m in job.follow.intersection(relevant):
dests.add(self.destinations[m])
if len(dests) > 1:
self.depending[msg_id] = job
self.fail_unreachable(msg_id)
return False
if job.targets:
# check blacklist+targets for impossibility
job.targets.difference_update(job.blacklist)
if not job.targets or not job.targets.intersection(
self.targets):
self.depending[msg_id] = job
self.fail_unreachable(msg_id)
return False
return False
else:
indices = None
self.submit_task(job, indices)
return True
def save_unmet(self, job):
"""Save a message for later submission when its dependencies are met."""
msg_id = job.msg_id
self.depending[msg_id] = job
# track the ids in follow or after, but not those already finished
for dep_id in job.after.union(job.follow).difference(self.all_done):
if dep_id not in self.graph:
self.graph[dep_id] = set()
self.graph[dep_id].add(msg_id)
def submit_task(self, job, indices=None):
"""Submit a task to any of a subset of our targets."""
if indices:
loads = [self.loads[i] for i in indices]
else:
loads = self.loads
idx = self.scheme(loads)
if indices:
idx = indices[idx]
target = self.targets[idx]
# print (target, map(str, msg[:3]))
# send job to the engine
self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False)
self.engine_stream.send_multipart(job.raw_msg, copy=False)
# update load
self.add_job(idx)
self.pending[target][job.msg_id] = job
# notify Hub
content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii'))
self.session.send(self.mon_stream,
'task_destination',
content=content,
ident=[b'tracktask', self.ident])
#-----------------------------------------------------------------------
# Result Handling
#-----------------------------------------------------------------------
@util.log_errors
def dispatch_result(self, raw_msg):
"""dispatch method for result replies"""
try:
idents, msg = self.session.feed_identities(raw_msg, copy=False)
msg = self.session.unserialize(msg, content=False, copy=False)
engine = idents[0]
try:
idx = self.targets.index(engine)
except ValueError:
pass # skip load-update for dead engines
else:
self.finish_job(idx)
except Exception:
self.log.error("task::Invaid result: %r", raw_msg, exc_info=True)
return
md = msg['metadata']
parent = msg['parent_header']
if md.get('dependencies_met', True):
success = (md['status'] == 'ok')
msg_id = parent['msg_id']
retries = self.retries[msg_id]
if not success and retries > 0:
# failed
self.retries[msg_id] = retries - 1
self.handle_unmet_dependency(idents, parent)
else:
del self.retries[msg_id]
# relay to client and update graph
self.handle_result(idents, parent, raw_msg, success)
# send to Hub monitor
self.mon_stream.send_multipart([b'outtask'] + raw_msg,
copy=False)
else:
self.handle_unmet_dependency(idents, parent)
def handle_result(self, idents, parent, raw_msg, success=True):
"""handle a real task result, either success or failure"""
# first, relay result to client
engine = idents[0]
client = idents[1]
# swap_ids for ROUTER-ROUTER mirror
raw_msg[:2] = [client, engine]
# print (map(str, raw_msg[:4]))
self.client_stream.send_multipart(raw_msg, copy=False)
# now, update our data structures
msg_id = parent['msg_id']
self.pending[engine].pop(msg_id)
if success:
self.completed[engine].add(msg_id)
self.all_completed.add(msg_id)
else:
self.failed[engine].add(msg_id)
self.all_failed.add(msg_id)
self.all_done.add(msg_id)
self.destinations[msg_id] = engine
self.update_graph(msg_id, success)
def handle_unmet_dependency(self, idents, parent):
"""handle an unmet dependency"""
engine = idents[0]
msg_id = parent['msg_id']
job = self.pending[engine].pop(msg_id)
job.blacklist.add(engine)
if job.blacklist == job.targets:
self.depending[msg_id] = job
self.fail_unreachable(msg_id)
elif not self.maybe_run(job):
# resubmit failed
if msg_id not in self.all_failed:
# put it back in our dependency tree
self.save_unmet(job)
if self.hwm:
try:
idx = self.targets.index(engine)
except ValueError:
pass # skip load-update for dead engines
else:
if self.loads[idx] == self.hwm - 1:
self.update_graph(None)
def update_graph(self, dep_id=None, success=True):
"""dep_id just finished. Update our dependency
graph and submit any jobs that just became runable.
Called with dep_id=None to update entire graph for hwm, but without finishing
a task.
"""
# print ("\n\n***********")
# pprint (dep_id)
# pprint (self.graph)
# pprint (self.depending)
# pprint (self.all_completed)
# pprint (self.all_failed)
# print ("\n\n***********\n\n")
# update any jobs that depended on the dependency
jobs = self.graph.pop(dep_id, [])
# recheck *all* jobs if
# a) we have HWM and an engine just become no longer full
# or b) dep_id was given as None
if dep_id is None or self.hwm and any(
[load == self.hwm - 1 for load in self.loads]):
jobs = self.depending.keys()
for msg_id in sorted(
jobs, key=lambda msg_id: self.depending[msg_id].timestamp):
job = self.depending[msg_id]
if job.after.unreachable(self.all_completed, self.all_failed)\
or job.follow.unreachable(self.all_completed, self.all_failed):
self.fail_unreachable(msg_id)
elif job.after.check(self.all_completed,
self.all_failed): # time deps met, maybe run
if self.maybe_run(job):
self.depending.pop(msg_id)
for mid in job.dependents:
if mid in self.graph:
self.graph[mid].remove(msg_id)
#----------------------------------------------------------------------
# methods to be overridden by subclasses
#----------------------------------------------------------------------
def add_job(self, idx):
"""Called after self.targets[idx] just got the job with header.
Override with subclasses. The default ordering is simple LRU.
The default loads are the number of outstanding jobs."""
self.loads[idx] += 1
for lis in (self.targets, self.loads):
lis.append(lis.pop(idx))
def finish_job(self, idx):
"""Called after self.targets[idx] just finished a job.
Override with subclasses."""
self.loads[idx] -= 1
class NotebookApp(BaseIPythonApplication):
name = 'ipython-notebook'
default_config_file_name='ipython_notebook_config.py'
description = """
The IPython HTML Notebook.
This launches a Tornado based HTML Notebook Server that serves up an
HTML5/Javascript Notebook client.
"""
examples = _examples
classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session,
MappingKernelManager, NotebookManager]
flags = Dict(flags)
aliases = Dict(aliases)
kernel_argv = List(Unicode)
log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
default_value=logging.INFO,
config=True,
help="Set the log level by value or name.")
# create requested profiles by default, if they don't exist:
auto_create = Bool(True)
# Network related information.
ip = Unicode(LOCALHOST, config=True,
help="The IP address the notebook server will listen on."
)
def _ip_changed(self, name, old, new):
if new == u'*': self.ip = u''
port = Integer(8888, config=True,
help="The port the notebook server will listen on."
)
certfile = Unicode(u'', config=True,
help="""The full path to an SSL/TLS certificate file."""
)
keyfile = Unicode(u'', config=True,
help="""The full path to a private key file for usage with SSL/TLS."""
)
password = Unicode(u'', config=True,
help="""Hashed password to use for web authentication.
To generate, type in a python/IPython shell:
from IPython.lib import passwd; passwd()
The string should be of the form type:salt:hashed-password.
"""
)
open_browser = Bool(True, config=True,
help="Whether to open in a browser after starting.")
read_only = Bool(False, config=True,
help="Whether to prevent editing/execution of notebooks."
)
webapp_settings = Dict(config=True,
help="Supply overrides for the tornado.web.Application that the "
"IPython notebook uses.")
enable_mathjax = Bool(True, config=True,
help="""Whether to enable MathJax for typesetting math/TeX
MathJax is the javascript library IPython uses to render math/LaTeX. It is
very large, so you may want to disable it if you have a slow internet
connection, or for offline use of the notebook.
When disabled, equations etc. will appear as their untransformed TeX source.
"""
)
def _enable_mathjax_changed(self, name, old, new):
"""set mathjax url to empty if mathjax is disabled"""
if not new:
self.mathjax_url = u''
mathjax_url = Unicode("", config=True,
help="""The url for MathJax.js."""
)
def _mathjax_url_default(self):
if not self.enable_mathjax:
return u''
static_path = self.webapp_settings.get("static_path", os.path.join(os.path.dirname(__file__), "static"))
if os.path.exists(os.path.join(static_path, 'mathjax', "MathJax.js")):
self.log.info("Using local MathJax")
return u"/static/mathjax/MathJax.js"
else:
self.log.info("Using MathJax from CDN")
return u"http://cdn.mathjax.org/mathjax/latest/MathJax.js"
def _mathjax_url_changed(self, name, old, new):
if new and not self.enable_mathjax:
# enable_mathjax=False overrides mathjax_url
self.mathjax_url = u''
else:
self.log.info("Using MathJax: %s", new)
def parse_command_line(self, argv=None):
super(NotebookApp, self).parse_command_line(argv)
if argv is None:
argv = sys.argv[1:]
# Scrub frontend-specific flags
self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
# Kernel should inherit default config file from frontend
self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
def init_configurables(self):
# force Session default to be secure
default_secure(self.config)
# Create a KernelManager and start a kernel.
self.kernel_manager = MappingKernelManager(
config=self.config, log=self.log, kernel_argv=self.kernel_argv,
connection_dir = self.profile_dir.security_dir,
)
self.notebook_manager = NotebookManager(config=self.config, log=self.log)
self.notebook_manager.list_notebooks()
def init_logging(self):
super(NotebookApp, self).init_logging()
# This prevents double log messages because tornado use a root logger that
# self.log is a child of. The logging module dipatches log messages to a log
# and all of its ancenstors until propagate is set to False.
self.log.propagate = False
def init_webapp(self):
"""initialize tornado webapp and httpserver"""
self.web_app = NotebookWebApplication(
self, self.kernel_manager, self.notebook_manager, self.log,
self.webapp_settings
)
if self.certfile:
ssl_options = dict(certfile=self.certfile)
if self.keyfile:
ssl_options['keyfile'] = self.keyfile
else:
ssl_options = None
self.web_app.password = self.password
self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
if ssl_options is None and not self.ip and not (self.read_only and not self.password):
self.log.critical('WARNING: the notebook server is listening on all IP addresses '
'but not using any encryption or authentication. This is highly '
'insecure and not recommended.')
# Try random ports centered around the default.
from random import randint
n = 50 # Max number of attempts, keep reasonably large.
for port in range(self.port, self.port+5) + [self.port + randint(-2*n, 2*n) for i in range(n-5)]:
try:
self.http_server.listen(port, self.ip)
except socket.error, e:
if e.errno != errno.EADDRINUSE:
raise
self.log.info('The port %i is already in use, trying another random port.' % port)
else:
self.port = port
break
class Status(Reference):
execution_state = Enum(('busy', 'idle'))