class HistoryManager(HistoryAccessor): """A class to organize all history-related functionality in one place. """ # Public interface # An instance of the IPython shell we are attached to shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') # Lists to hold processed and raw history. These start with a blank entry # so that we can index them starting from 1 input_hist_parsed = List([""]) input_hist_raw = List([""]) # A list of directories visited during session dir_hist = List() def _dir_hist_default(self): try: return [os.getcwdu()] except OSError: return [] # A dict of output history, keyed with ints from the shell's # execution count. output_hist = Dict() # The text/plain repr of outputs. output_hist_reprs = Dict() # The number of the current session in the history database session_number = Integer() # Should we log output to the database? (default no) db_log_output = Bool(False, config=True) # Write to database every x commands (higher values save disk access & power) # Values of 1 or less effectively disable caching. db_cache_size = Integer(0, config=True) # The input and output caches db_input_cache = List() db_output_cache = List() # History saving in separate thread save_thread = Instance('IPython.core.history.HistorySavingThread') try: # Event is a function returning an instance of _Event... save_flag = Instance(threading._Event) except AttributeError: # ...until Python 3.3, when it's a class. save_flag = Instance(threading.Event) # Private interface # Variables used to store the three last inputs from the user. On each new # history update, we populate the user's namespace with these, shifted as # necessary. _i00 = Unicode(u'') _i = Unicode(u'') _ii = Unicode(u'') _iii = Unicode(u'') # A regex matching all forms of the exit command, so that we don't store # them in the history (it's annoying to rewind the first entry and land on # an exit call). _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") def __init__(self, shell=None, config=None, **traits): """Create a new history manager associated with a shell instance. """ # We need a pointer back to the shell for various tasks. super(HistoryManager, self).__init__(shell=shell, config=config, **traits) self.save_flag = threading.Event() self.db_input_cache_lock = threading.Lock() self.db_output_cache_lock = threading.Lock() if self.enabled and self.hist_file != ':memory:': self.save_thread = HistorySavingThread(self) self.save_thread.start() self.new_session() def _get_hist_file_name(self, profile=None): """Get default history file name based on the Shell's profile. The profile parameter is ignored, but must exist for compatibility with the parent class.""" profile_dir = self.shell.profile_dir.location return os.path.join(profile_dir, 'history.sqlite') @needs_sqlite def new_session(self, conn=None): """Get a new session number.""" if conn is None: conn = self.db with conn: cur = conn.execute( """INSERT INTO sessions VALUES (NULL, ?, NULL, NULL, "") """, (datetime.datetime.now(), )) self.session_number = cur.lastrowid def end_session(self): """Close the database session, filling in the end time and line count.""" self.writeout_cache() with self.db: self.db.execute( """UPDATE sessions SET end=?, num_cmds=? WHERE session==?""", (datetime.datetime.now(), len(self.input_hist_parsed) - 1, self.session_number)) self.session_number = 0 def name_session(self, name): """Give the current session a name in the history database.""" with self.db: self.db.execute("UPDATE sessions SET remark=? WHERE session==?", (name, self.session_number)) def reset(self, new_session=True): """Clear the session history, releasing all object references, and optionally open a new session.""" self.output_hist.clear() # The directory history can't be completely empty self.dir_hist[:] = [os.getcwdu()] if new_session: if self.session_number: self.end_session() self.input_hist_parsed[:] = [""] self.input_hist_raw[:] = [""] self.new_session() # ------------------------------ # Methods for retrieving history # ------------------------------ def _get_range_session(self, start=1, stop=None, raw=True, output=False): """Get input and output history from the current session. Called by get_range, and takes similar parameters.""" input_hist = self.input_hist_raw if raw else self.input_hist_parsed n = len(input_hist) if start < 0: start += n if not stop or (stop > n): stop = n elif stop < 0: stop += n for i in range(start, stop): if output: line = (input_hist[i], self.output_hist_reprs.get(i)) else: line = input_hist[i] yield (0, i, line) def get_range(self, session=0, start=1, stop=None, raw=True, output=False): """Retrieve input by session. Parameters ---------- session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is previous session. start : int First line to retrieve. stop : int End of line range (excluded from output itself). If None, retrieve to the end of the session. raw : bool If True, return untranslated input output : bool If True, attempt to include output. This will be 'real' Python objects for the current session, or text reprs from previous sessions if db_log_output was enabled at the time. Where no output is found, None is used. Returns ------- An iterator over the desired lines. Each line is a 3-tuple, either (session, line, input) if output is False, or (session, line, (input, output)) if output is True. """ if session <= 0: session += self.session_number if session == self.session_number: # Current session return self._get_range_session(start, stop, raw, output) return super(HistoryManager, self).get_range(session, start, stop, raw, output) ## ---------------------------- ## Methods for storing history: ## ---------------------------- def store_inputs(self, line_num, source, source_raw=None): """Store source and raw input in history and create input cache variables _i*. Parameters ---------- line_num : int The prompt number of this input. source : str Python input. source_raw : str, optional If given, this is the raw input without any IPython transformations applied to it. If not given, ``source`` is used. """ if source_raw is None: source_raw = source source = source.rstrip('\n') source_raw = source_raw.rstrip('\n') # do not store exit/quit commands if self._exit_re.match(source_raw.strip()): return self.input_hist_parsed.append(source) self.input_hist_raw.append(source_raw) with self.db_input_cache_lock: self.db_input_cache.append((line_num, source, source_raw)) # Trigger to flush cache and write to DB. if len(self.db_input_cache) >= self.db_cache_size: self.save_flag.set() # update the auto _i variables self._iii = self._ii self._ii = self._i self._i = self._i00 self._i00 = source_raw # hackish access to user namespace to create _i1,_i2... dynamically new_i = '_i%s' % line_num to_main = { '_i': self._i, '_ii': self._ii, '_iii': self._iii, new_i: self._i00 } if self.shell is not None: self.shell.push(to_main, interactive=False) def store_output(self, line_num): """If database output logging is enabled, this saves all the outputs from the indicated prompt number to the database. It's called by run_cell after code has been executed. Parameters ---------- line_num : int The line number from which to save outputs """ if (not self.db_log_output) or (line_num not in self.output_hist_reprs): return output = self.output_hist_reprs[line_num] with self.db_output_cache_lock: self.db_output_cache.append((line_num, output)) if self.db_cache_size <= 1: self.save_flag.set() def _writeout_input_cache(self, conn): with conn: for line in self.db_input_cache: conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)", (self.session_number, ) + line) def _writeout_output_cache(self, conn): with conn: for line in self.db_output_cache: conn.execute("INSERT INTO output_history VALUES (?, ?, ?)", (self.session_number, ) + line) @needs_sqlite def writeout_cache(self, conn=None): """Write any entries in the cache to the database.""" if conn is None: conn = self.db with self.db_input_cache_lock: try: self._writeout_input_cache(conn) except sqlite3.IntegrityError: self.new_session(conn) print("ERROR! Session/line number was not unique in", "database. History logging moved to new session", self.session_number) try: # Try writing to the new session. If this fails, don't # recurse self._writeout_input_cache(conn) except sqlite3.IntegrityError: pass finally: self.db_input_cache = [] with self.db_output_cache_lock: try: self._writeout_output_cache(conn) except sqlite3.IntegrityError: print("!! Session/line number for output was not unique", "in database. Output will not be stored.") finally: self.db_output_cache = []
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples aliases = aliases flags = flags classes = [ KernelManager, ProfileDir, Session, MappingKernelManager, ContentsManager, FileContentsManager, NotebookNotary, ] flags = Dict(flags) aliases = Dict(aliases) subcommands = dict( list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), ) ipython_kernel_argv = List(Unicode) _log_formatter_cls = LogFormatter def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%H:%M:%S" def _log_format_default(self): """override default log format to include time""" return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('', config=True) def _file_to_run_changed(self, name, old, new): path, base = os.path.split(new) if path: self.file_to_run = base self.notebook_dir = path # Network related information allow_origin = Unicode('', config=True, help="""Set the Access-Control-Allow-Origin header Use '*' to allow any origin to access your server. Takes precedence over allow_origin_pat. """ ) allow_origin_pat = Unicode('', config=True, help="""Use a regular expression for the Access-Control-Allow-Origin header Requests from an origin matching the expression will get replies with: Access-Control-Allow-Origin: origin where `origin` is the origin of the request. Ignored if allow_origin is set. """ ) allow_credentials = Bool(False, config=True, help="Set the Access-Control-Allow-Credentials: true header" ) default_url = Unicode('/tree', config=True, help="The default URL to redirect to from `/`" ) ip = Unicode('localhost', config=True, help="The IP address the notebook server will listen on." ) def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on." ) port_retries = Integer(50, config=True, help="The number of additional ports to try if the specified port is not available." ) certfile = Unicode(u'', config=True, help="""The full path to an SSL/TLS certificate file.""" ) keyfile = Unicode(u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""" ) cookie_secret_file = Unicode(config=True, help="""The file where the cookie secret is stored.""" ) def _cookie_secret_file_default(self): if self.profile_dir is None: return '' return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret') cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """ ) def _cookie_secret_default(self): if os.path.exists(self.cookie_secret_file): with io.open(self.cookie_secret_file, 'rb') as f: return f.read() else: secret = base64.encodestring(os.urandom(1024)) self._write_cookie_secret_file(secret) return secret def _write_cookie_secret_file(self, secret): """write my secret to my secret_file""" self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file) with io.open(self.cookie_secret_file, 'wb') as f: f.write(secret) try: os.chmod(self.cookie_secret_file, 0o600) except OSError: self.log.warn( "Could not set permissions on %s", self.cookie_secret_file ) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """ ) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict(config=True, help="DEPRECATED, use tornado_settings" ) def _webapp_settings_changed(self, name, old, new): self.log.warn("\n webapp_settings is deprecated, use tornado_settings.\n") self.tornado_settings = new tornado_settings = Dict(config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") jinja_environment_options = Dict(config=True, help="Supply extra arguments that will be passed to Jinja environment.") enable_mathjax = Bool(True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """ ) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_url_changed(self, name, old, new): if not new.startswith('/'): self.base_url = '/'+new elif not new.endswith('/'): self.base_url = new+'/' base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""") def _base_project_url_changed(self, name, old, new): self.log.warn("base_project_url is deprecated, use base_url") self.base_url = new extra_static_paths = List(Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""" ) def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] extra_template_paths = List(Unicode, config=True, help="""Extra paths to search for serving jinja templates. Can be used to override templates from IPython.html.templates.""" ) def _extra_template_paths_default(self): return [] @property def template_file_path(self): """return extra paths + the default locations""" return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST nbextensions_path = List(Unicode, config=True, help="""paths for Javascript extensions. By default, this is just IPYTHONDIR/nbextensions""" ) def _nbextensions_path_default(self): return [os.path.join(get_ipython_dir(), 'nbextensions')] websocket_url = Unicode("", config=True, help="""The base URL for websockets, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """ ) mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""" ) def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.tornado_settings.get("static_url_prefix", url_path_join(self.base_url, "static") ) # try local mathjax, either in nbextensions/mathjax or static/mathjax for (url_prefix, search_path) in [ (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path), (static_url_prefix, self.static_file_path), ]: self.log.debug("searching for local mathjax in %s", search_path) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path) except IOError: continue else: url = url_path_join(url_prefix, u"mathjax/MathJax.js") self.log.info("Serving local MathJax from %s at %s", mathjax, url) return url # no local mathjax, serve from CDN url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) contents_manager_class = DottedObjectName('IPython.html.services.contents.filemanager.FileContentsManager', config=True, help='The notebook manager class to use.' ) kernel_manager_class = DottedObjectName('IPython.html.services.kernels.kernelmanager.MappingKernelManager', config=True, help='The kernel manager class to use.' ) session_manager_class = DottedObjectName('IPython.html.services.sessions.sessionmanager.SessionManager', config=True, help='The session manager class to use.' ) cluster_manager_class = DottedObjectName('IPython.html.services.clusters.clustermanager.ClusterManager', config=True, help='The cluster manager class to use.' ) kernel_spec_manager = Instance(KernelSpecManager) def _kernel_spec_manager_default(self): return KernelSpecManager(ipython_dir=self.ipython_dir) trust_xheaders = Bool(False, config=True, help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL") ) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json"%os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) notebook_dir = Unicode(py3compat.getcwd(), config=True, help="The directory to use for notebooks and kernels." ) pylab = Unicode('disabled', config=True, help=""" DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. """ ) def _pylab_changed(self, name, old, new): """when --pylab is specified, display a warning and exit""" if new != 'warn': backend = ' %s' % new else: backend = '' self.log.error("Support for specifying --pylab on the command line has been removed.") self.log.error( "Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself.".format(backend) ) self.exit(1) def _notebook_dir_changed(self, name, old, new): """Do a bit of validation of the notebook dir.""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. self.notebook_dir = os.path.abspath(new) return if not os.path.isdir(new): raise TraitError("No such notebook dir: %r" % new) # setting App.notebook_dir implies setting notebook and kernel dirs as well self.config.FileContentsManager.root_dir = new self.config.MappingKernelManager.root_dir = new def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: arg0 = self.extra_args[0] f = os.path.abspath(arg0) self.argv.remove(arg0) if not os.path.exists(f): self.log.critical("No such file or directory: %s", f) self.exit(1) # Use config here, to ensure that it takes higher priority than # anything that comes from the profile. c = Config() if os.path.isdir(f): c.NotebookApp.notebook_dir = f elif os.path.isfile(f): c.NotebookApp.file_to_run = f self.update_config(c) def init_kernel_argv(self): """add the profile-dir to arguments to be passed to IPython kernels""" # FIXME: remove special treatment of IPython kernels # Kernel should get *absolute* path to profile directory self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location] def init_configurables(self): # force Session default to be secure default_secure(self.config) kls = import_item(self.kernel_manager_class) self.kernel_manager = kls( parent=self, log=self.log, ipython_kernel_argv=self.ipython_kernel_argv, connection_dir = self.profile_dir.security_dir, ) kls = import_item(self.contents_manager_class) self.contents_manager = kls(parent=self, log=self.log) kls = import_item(self.session_manager_class) self.session_manager = kls(parent=self, log=self.log, kernel_manager=self.kernel_manager, contents_manager=self.contents_manager) kls = import_item(self.cluster_manager_class) self.cluster_manager = kls(parent=self, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False for log in app_log, access_log, gen_log: # consistent log output name (NotebookApp instead of tornado.access, etc.) log.name = self.log.name # hook up tornado 3's loggers to our app handlers logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.tornado_settings['allow_origin'] = self.allow_origin if self.allow_origin_pat: self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat) self.tornado_settings['allow_credentials'] = self.allow_credentials self.web_app = NotebookWebApplication( self, self.kernel_manager, self.contents_manager, self.cluster_manager, self.session_manager, self.kernel_spec_manager, self.log, self.base_url, self.default_url, self.tornado_settings, self.jinja_environment_options ) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical(warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries+1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info('The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical('ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else 'localhost' return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) def init_terminals(self): try: from .terminal import initialize initialize(self.web_app) self.web_app.settings['terminals_available'] = True except ImportError as e: self.log.info("Terminals not available (error was %s)", e) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r,w,x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y') and 'n' not in line.lower(): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn("components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn("components submodule unclean, you may see 404s on static/components") self.log.warn("run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_terminals() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.contents_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return {'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), 'base_url': self.base_url, 'notebook_dir': os.path.abspath(self.notebook_dir), 'pid': os.getpid() } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).") self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: fullpath = os.path.join(self.notebook_dir, self.file_to_run) if not os.path.exists(fullpath): self.log.critical("%s does not exist" % fullpath) self.exit(1) uri = url_path_join('notebooks', self.file_to_run) else: uri = 'tree' if browser: b = lambda : browser.open(url_path_join(self.connection_url, uri), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file()
class UnionListTrait(HasTraits): value = List(Int() | Bool())
class StoreMagics(Magics): """Lightweight persistence for python variables. Provides the %store magic.""" autorestore = Bool( False, config=True, help="""If True, any %store-d variables will be automatically restored when IPython starts. """) def __init__(self, shell): super(StoreMagics, self).__init__(shell=shell) self.shell.configurables.append(self) if self.autorestore: restore_data(self.shell) @skip_doctest @line_magic def store(self, parameter_s=''): """Lightweight persistence for python variables. Example:: In [1]: l = ['hello',10,'world'] In [2]: %store l In [3]: exit (IPython session is closed and started again...) ville@badger:~$ ipython In [1]: l NameError: name 'l' is not defined In [2]: %store -r In [3]: l Out[3]: ['hello', 10, 'world'] Usage: * ``%store`` - Show list of all variables and their current values * ``%store spam`` - Store the *current* value of the variable spam to disk * ``%store -d spam`` - Remove the variable and its value from storage * ``%store -z`` - Remove all variables from storage * ``%store -r`` - Refresh all variables from store (overwrite current vals) * ``%store -r spam bar`` - Refresh specified variables from store (delete current val) * ``%store foo >a.txt`` - Store value of foo to new file a.txt * ``%store foo >>a.txt`` - Append value of foo to file a.txt It should be noted that if you change the value of a variable, you need to %store it again if you want to persist the new value. Note also that the variables will need to be pickleable; most basic python types can be safely %store'd. Also aliases can be %store'd across sessions. """ opts, argsl = self.parse_options(parameter_s, 'drz', mode='string') args = argsl.split(None, 1) ip = self.shell db = ip.db # delete if 'd' in opts: try: todel = args[0] except IndexError: raise UsageError('You must provide the variable to forget') else: try: del db['autorestore/' + todel] except: raise UsageError("Can't delete variable '%s'" % todel) # reset elif 'z' in opts: for k in db.keys('autorestore/*'): del db[k] elif 'r' in opts: if args: for arg in args: try: obj = db['autorestore/' + arg] except KeyError: print("no stored variable %s" % arg) else: ip.user_ns[arg] = obj else: restore_data(ip) # run without arguments -> list variables & values elif not args: vars = db.keys('autorestore/*') vars.sort() if vars: size = max(map(len, vars)) else: size = 0 print('Stored variables and their in-db values:') fmt = '%-' + str(size) + 's -> %s' get = db.get for var in vars: justkey = os.path.basename(var) # print 30 first characters from every var print(fmt % (justkey, repr(get(var, '<unavailable>'))[:50])) # default action - store the variable else: # %store foo >file.txt or >>file.txt if len(args) > 1 and args[1].startswith('>'): fnam = os.path.expanduser(args[1].lstrip('>').lstrip()) if args[1].startswith('>>'): fil = open(fnam, 'a') else: fil = open(fnam, 'w') obj = ip.ev(args[0]) print("Writing '%s' (%s) to file '%s'." % (args[0], obj.__class__.__name__, fnam)) if not isinstance(obj, string_types): from pprint import pprint pprint(obj, fil) else: fil.write(obj) if not obj.endswith('\n'): fil.write('\n') fil.close() return # %store foo try: obj = ip.user_ns[args[0]] except KeyError: # it might be an alias name = args[0] try: cmd = ip.alias_manager.retrieve_alias(name) except ValueError: raise UsageError("Unknown variable '%s'" % name) staliases = db.get('stored_aliases', {}) staliases[name] = cmd db['stored_aliases'] = staliases print("Alias stored: %s (%s)" % (name, cmd)) return else: modname = getattr(inspect.getmodule(obj), '__name__', '') if modname == '__main__': print( textwrap.dedent("""\ Warning:%s is %s Proper storage of interactively declared classes (or instances of those classes) is not possible! Only instances of classes in real modules on file system can be %%store'd. """ % (args[0], obj))) return #pickled = pickle.dumps(obj) db['autorestore/' + args[0]] = obj print("Stored '%s' (%s)" % (args[0], obj.__class__.__name__))
class ListApp(TransferApp): name = u'nbgrader-list' description = u'List assignments in the nbgrader exchange' aliases = aliases flags = flags examples = """ List assignments in the nbgrader exchange. For the usage of instructors and students. Students ======== To list assignments for a course, you must first know the `course_id` for your course. If you don't know it, ask your instructor. To list the released assignments for the course `phys101`: nbgrader list phys101 Instructors =========== To list outbound (released) or inbound (submitted) assignments for a course, you must configure the `course_id` in your config file or the command line. To see all of the released assignments, run nbgrader list # course_id in the config file or nbgrader list --course phys101 # course_id provided To see the inbound (submitted) assignments: nbgrader list --inbound phys101 You can use the `--student` and `--assignment` options to filter the list by student or assignment: nbgrader list --inbound --student=student1 --assignment=assignment1 phys101 If a student has submitted an assignment multiple times, the `list` command will show all submissions with their timestamps. The `list` command can optionally remove listed assignments by providing the `--remove` flag: nbgrader list --inbound --remove --student=student1 phys101 """ inbound = Bool(False, config=True, help="List inbound files rather than outbound.") remove = Bool(False, config=True, help="Remove, rather than list files.") def init_src(self): pass def init_dest(self): self.course_path = os.path.join(self.exchange_directory, self.course_id) self.outbound_path = os.path.join(self.course_path, 'outbound') self.inbound_path = os.path.join(self.course_path, 'inbound') assignment_id = self.assignment_id if self.assignment_id else '*' if self.inbound: student_id = self.student_id if self.student_id else '*' pattern = os.path.join(self.inbound_path, '{}+{}+*'.format(student_id, assignment_id)) else: pattern = os.path.join(self.outbound_path, assignment_id) self.assignments = sorted(glob.glob(pattern)) def copy_files(self): pass def list_files(self): """List files.""" if self.inbound: self.log.info("Submitted assignments:") for path in self.assignments: username, assignment, timestamp = os.path.split(path)[1].rsplit('+',2) self.log.info("{} {} {} {}".format( self.course_id, username, assignment, timestamp )) else: self.log.info("Released assignments:") for path in self.assignments: self.log.info("{} {}".format(self.course_id, os.path.split(path)[1])) def remove_files(self): """List and remove files.""" if self.inbound: self.log.info("Removing submitted assignments:") for path in self.assignments: username, assignment, timestamp = os.path.split(path)[1].rsplit('+',2) self.log.info("{} {} {} {}".format( self.course_id, username, assignment, timestamp )) shutil.rmtree(path) else: self.log.info("Removing released assignments:") for path in self.assignments: self.log.info("{} {}".format(self.course_id, os.path.split(path)[1])) shutil.rmtree(path) def start(self): if len(self.extra_args) == 0: self.extra_args = ["*"] # allow user to not put in assignment super(ListApp, self).start() if self.remove: self.remove_files() else: self.list_files()
class InteractiveShellApp(Configurable): """A Mixin for applications that start InteractiveShell instances. Provides configurables for loading extensions and executing files as part of configuring a Shell environment. The following methods should be called by the :meth:`initialize` method of the subclass: - :meth:`init_path` - :meth:`init_shell` (to be implemented by the subclass) - :meth:`init_gui_pylab` - :meth:`init_extensions` - :meth:`init_code` """ extensions = List( Unicode, config=True, help="A list of dotted module names of IPython extensions to load.") extra_extension = Unicode( '', config=True, help="dotted module name of an IPython extension to load.") reraise_ipython_extension_failures = Bool( False, config=True, help="Reraise exceptions encountered loading IPython extensions?", ) # Extensions that are always loaded (not configurable) default_extensions = List(Unicode, [u'storemagic'], config=False) hide_initial_ns = Bool( True, config=True, help= """Should variables loaded at startup (by startup files, exec_lines, etc.) be hidden from tools like %who?""") exec_files = List(Unicode, config=True, help="""List of files to run at IPython startup.""") exec_PYTHONSTARTUP = Bool( True, config=True, help="""Run the file referenced by the PYTHONSTARTUP environment variable at IPython startup.""") file_to_run = Unicode('', config=True, help="""A file to be run""") exec_lines = List(Unicode, config=True, help="""lines of code to run at IPython startup.""") code_to_run = Unicode('', config=True, help="Execute the given command string.") module_to_run = Unicode('', config=True, help="Run the module as a script.") gui = CaselessStrEnum( gui_keys, config=True, allow_none=True, help="Enable GUI event loop integration with any of {0}.".format( gui_keys)) matplotlib = CaselessStrEnum( backend_keys, allow_none=True, config=True, help="""Configure matplotlib for interactive use with the default matplotlib backend.""") pylab = CaselessStrEnum( backend_keys, allow_none=True, config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """) pylab_import_all = Bool( True, config=True, help= """If true, IPython will populate the user namespace with numpy, pylab, etc. and an ``import *`` is done from numpy and pylab, when using pylab mode. When False, pylab mode should not import any names into the user namespace. """) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() def init_path(self): """Add current working directory, '', to sys.path""" if sys.path[0] != '': sys.path.insert(0, '') def init_shell(self): raise NotImplementedError("Override in subclasses") def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" enable = False shell = self.shell if self.pylab: enable = lambda key: shell.enable_pylab( key, import_all=self.pylab_import_all) key = self.pylab elif self.matplotlib: enable = shell.enable_matplotlib key = self.matplotlib elif self.gui: enable = shell.enable_gui key = self.gui if not enable: return try: r = enable(key) except ImportError: self.log.warn( "Eventloop or matplotlib integration failed. Is matplotlib installed?" ) self.shell.showtraceback() return except Exception: self.log.warn("GUI event loop or pylab initialization failed") self.shell.showtraceback() return if isinstance(r, tuple): gui, backend = r[:2] self.log.info( "Enabling GUI event loop integration, " "eventloop=%s, matplotlib=%s", gui, backend) if key == "auto": print("Using matplotlib backend: %s" % backend) else: gui = r self.log.info( "Enabling GUI event loop integration, " "eventloop=%s", gui) def init_extensions(self): """Load all IPython extensions in IPythonApp.extensions. This uses the :meth:`ExtensionManager.load_extensions` to load all the extensions listed in ``self.extensions``. """ try: self.log.debug("Loading IPython extensions...") extensions = self.default_extensions + self.extensions if self.extra_extension: extensions.append(self.extra_extension) for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) self.shell.extension_manager.load_extension(ext) except: if self.reraise_ipython_extension_failures: raise msg = ("Error in loading extension: {ext}\n" "Check your config files in {location}".format( ext=ext, location=self.profile_dir.location)) self.log.warn(msg, exc_info=True) except: if self.reraise_ipython_extension_failures: raise self.log.warn("Unknown error in loading extensions:", exc_info=True) def init_code(self): """run the pre-flight code, specified via exec_lines""" self._run_startup_files() self._run_exec_lines() self._run_exec_files() # Hide variables defined here from %who etc. if self.hide_initial_ns: self.shell.user_ns_hidden.update(self.shell.user_ns) # command-line execution (ipython -i script.py, ipython -m module) # should *not* be excluded from %whos self._run_cmd_line_code() self._run_module() # flush output, so itwon't be attached to the first cell sys.stdout.flush() sys.stderr.flush() def _run_exec_lines(self): """Run lines of code in IPythonApp.exec_lines in the user's namespace.""" if not self.exec_lines: return try: self.log.debug("Running code from IPythonApp.exec_lines...") for line in self.exec_lines: try: self.log.info("Running code in user namespace: %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user " "namespace: %s" % line) self.shell.showtraceback() except: self.log.warn("Unknown error in handling IPythonApp.exec_lines:") self.shell.showtraceback() def _exec_file(self, fname, shell_futures=False): try: full_filename = filefind(fname, [u'.', self.ipython_dir]) except IOError as e: self.log.warn("File not found: %r" % fname) return # Make sure that the running script gets a proper sys.argv as if it # were run from a system shell. save_argv = sys.argv sys.argv = [full_filename] + self.extra_args[1:] # protect sys.argv from potential unicode strings on Python 2: if not py3compat.PY3: sys.argv = [py3compat.cast_bytes(a) for a in sys.argv] try: if os.path.isfile(full_filename): self.log.info("Running file in user namespace: %s" % full_filename) # Ensure that __file__ is always defined to match Python # behavior. with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = fname if full_filename.endswith('.ipy'): self.shell.safe_execfile_ipy( full_filename, shell_futures=shell_futures) else: # default to python, even without extension self.shell.safe_execfile(full_filename, self.shell.user_ns, shell_futures=shell_futures) finally: sys.argv = save_argv def _run_startup_files(self): """Run files from profile startup directory""" startup_dir = self.profile_dir.startup_dir startup_files = [] if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \ not (self.file_to_run or self.code_to_run or self.module_to_run): python_startup = os.environ['PYTHONSTARTUP'] self.log.debug("Running PYTHONSTARTUP file %s...", python_startup) try: self._exec_file(python_startup) except: self.log.warn( "Unknown error in handling PYTHONSTARTUP file %s:", python_startup) self.shell.showtraceback() finally: # Many PYTHONSTARTUP files set up the readline completions, # but this is often at odds with IPython's own completions. # Do not allow PYTHONSTARTUP to set up readline. if self.shell.has_readline: self.shell.set_readline_completer() startup_files += glob.glob(os.path.join(startup_dir, '*.py')) startup_files += glob.glob(os.path.join(startup_dir, '*.ipy')) if not startup_files: return self.log.debug("Running startup files from %s...", startup_dir) try: for fname in sorted(startup_files): self._exec_file(fname) except: self.log.warn("Unknown error in handling startup files:") self.shell.showtraceback() def _run_exec_files(self): """Run files from IPythonApp.exec_files""" if not self.exec_files: return self.log.debug("Running files in IPythonApp.exec_files...") try: for fname in self.exec_files: self._exec_file(fname) except: self.log.warn("Unknown error in handling IPythonApp.exec_files:") self.shell.showtraceback() def _run_cmd_line_code(self): """Run code or file specified at the command-line""" if self.code_to_run: line = self.code_to_run try: self.log.info("Running code given at command line (c=): %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user namespace: %s" % line) self.shell.showtraceback() # Like Python itself, ignore the second if the first of these is present elif self.file_to_run: fname = self.file_to_run try: self._exec_file(fname, shell_futures=True) except: self.log.warn("Error in executing file in user namespace: %s" % fname) self.shell.showtraceback() def _run_module(self): """Run module specified at the command-line.""" if self.module_to_run: # Make sure that the module gets a proper sys.argv as if it were # run using `python -m`. save_argv = sys.argv sys.argv = [sys.executable] + self.extra_args try: self.shell.safe_run_module(self.module_to_run, self.shell.user_ns) finally: sys.argv = save_argv
class EngineFactory(RegistrationFactory): """IPython engine""" # configurables: out_stream_factory = Type('IPython.kernel.zmq.iostream.OutStream', config=True, help="""The OutStream for handling stdout/err. Typically 'IPython.kernel.zmq.iostream.OutStream'""") display_hook_factory = Type( 'IPython.kernel.zmq.displayhook.ZMQDisplayHook', config=True, help="""The class for handling displayhook. Typically 'IPython.kernel.zmq.displayhook.ZMQDisplayHook'""") location = Unicode( config=True, help="""The location (an IP address) of the controller. This is used for disambiguating URLs, to determine whether loopback should be used to connect or the public address.""") timeout = Float( 5.0, config=True, help="""The time (in seconds) to wait for the Controller to respond to registration requests before giving up.""") max_heartbeat_misses = Integer( 50, config=True, help="""The maximum number of times a check for the heartbeat ping of a controller can be missed before shutting down the engine. If set to 0, the check is disabled.""") sshserver = Unicode( config=True, help= """The SSH server to use for tunneling connections to the Controller.""" ) sshkey = Unicode( config=True, help= """The SSH private key file to use when tunneling connections to the Controller.""" ) paramiko = Bool( sys.platform == 'win32', config=True, help="""Whether to use paramiko instead of openssh for tunnels.""") # not configurable: connection_info = Dict() user_ns = Dict() id = Integer(allow_none=True) registrar = Instance('zmq.eventloop.zmqstream.ZMQStream') kernel = Instance(Kernel) hb_check_period = Integer() # States for the heartbeat monitoring # Initial values for monitored and pinged must satisfy "monitored > pinged == False" so that # during the first check no "missed" ping is reported. Must be floats for Python 3 compatibility. _hb_last_pinged = 0.0 _hb_last_monitored = 0.0 _hb_missed_beats = 0 # The zmq Stream which receives the pings from the Heart _hb_listener = None bident = CBytes() ident = Unicode() def _ident_changed(self, name, old, new): self.bident = cast_bytes(new) using_ssh = Bool(False) def __init__(self, **kwargs): super(EngineFactory, self).__init__(**kwargs) self.ident = self.session.session def init_connector(self): """construct connection function, which handles tunnels.""" self.using_ssh = bool(self.sshkey or self.sshserver) if self.sshkey and not self.sshserver: # We are using ssh directly to the controller, tunneling localhost to localhost self.sshserver = self.url.split('://')[1].split(':')[0] if self.using_ssh: if tunnel.try_passwordless_ssh(self.sshserver, self.sshkey, self.paramiko): password = False else: password = getpass("SSH Password for %s: " % self.sshserver) else: password = False def connect(s, url): url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) return tunnel.tunnel_connection( s, url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) else: return s.connect(url) def maybe_tunnel(url): """like connect, but don't complete the connection (for use by heartbeat)""" url = disambiguate_url(url, self.location) if self.using_ssh: self.log.debug("Tunneling connection to %s via %s", url, self.sshserver) url, tunnelobj = tunnel.open_tunnel( url, self.sshserver, keyfile=self.sshkey, paramiko=self.paramiko, password=password, ) return str(url) return connect, maybe_tunnel def register(self): """send the registration_request""" self.log.info("Registering with controller at %s" % self.url) ctx = self.context connect, maybe_tunnel = self.init_connector() reg = ctx.socket(zmq.DEALER) reg.setsockopt(zmq.IDENTITY, self.bident) connect(reg, self.url) self.registrar = zmqstream.ZMQStream(reg, self.loop) content = dict(uuid=self.ident) self.registrar.on_recv( lambda msg: self.complete_registration(msg, connect, maybe_tunnel)) # print (self.session.key) self.session.send(self.registrar, "registration_request", content=content) def _report_ping(self, msg): """Callback for when the heartmonitor.Heart receives a ping""" #self.log.debug("Received a ping: %s", msg) self._hb_last_pinged = time.time() def complete_registration(self, msg, connect, maybe_tunnel): # print msg self._abort_dc.stop() ctx = self.context loop = self.loop identity = self.bident idents, msg = self.session.feed_identities(msg) msg = self.session.unserialize(msg) content = msg['content'] info = self.connection_info def url(key): """get zmq url for given channel""" return str(info["interface"] + ":%i" % info[key]) if content['status'] == 'ok': self.id = int(content['id']) # launch heartbeat # possibly forward hb ports with tunnels hb_ping = maybe_tunnel(url('hb_ping')) hb_pong = maybe_tunnel(url('hb_pong')) hb_monitor = None if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) mport = mon.bind_to_random_port('tcp://%s' % LOCALHOST) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) hb_monitor = "tcp://%s:%i" % (LOCALHOST, mport) heart = Heart(hb_ping, hb_pong, hb_monitor, heart_id=identity) heart.start() # create Shell Connections (MUX, Task, etc.): shell_addrs = url('mux'), url('task') # Use only one shell stream for mux and tasks stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) stream.setsockopt(zmq.IDENTITY, identity) shell_streams = [stream] for addr in shell_addrs: connect(stream, addr) # control stream: control_addr = url('control') control_stream = zmqstream.ZMQStream(ctx.socket(zmq.ROUTER), loop) control_stream.setsockopt(zmq.IDENTITY, identity) connect(control_stream, control_addr) # create iopub stream: iopub_addr = url('iopub') iopub_socket = ctx.socket(zmq.PUB) iopub_socket.setsockopt(zmq.IDENTITY, identity) connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory( self.session, iopub_socket) sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id) self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.shell.display_pub.topic = cast_bytes( 'engine.%i.displaypub' % self.id) # periodically check the heartbeat pings of the controller # Should be started here and not in "start()" so that the right period can be taken # from the hubs HeartBeatMonitor.period if self.max_heartbeat_misses > 0: # Use a slightly bigger check period than the hub signal period to not warn unnecessary self.hb_check_period = int(content['hb_period']) + 10 self.log.info( "Starting to monitor the heartbeat signal from the hub every %i ms.", self.hb_check_period) self._hb_reporter = ioloop.PeriodicCallback( self._hb_monitor, self.hb_check_period, self.loop) self._hb_reporter.start() else: self.log.info( "Monitoring of the heartbeat signal from the hub is not enabled." ) # FIXME: This is a hack until IPKernelApp and IPEngineApp can be fully merged app = IPKernelApp(config=self.config, shell=self.kernel.shell, kernel=self.kernel, log=self.log) app.init_profile_dir() app.init_code() self.kernel.start() else: self.log.fatal("Registration Failed: %s" % msg) raise Exception("Registration Failed: %s" % msg) self.log.info("Completed registration with id %i" % self.id) def abort(self): self.log.fatal("Registration timed out after %.1f seconds" % self.timeout) if self.url.startswith('127.'): self.log.fatal(""" If the controller and engines are not on the same machine, you will have to instruct the controller to listen on an external IP (in ipcontroller_config.py): c.HubFactory.ip='*' # for all interfaces, internal and external c.HubFactory.ip='192.168.1.101' # or any interface that the engines can see or tunnel connections via ssh. """) self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) time.sleep(1) sys.exit(255) def _hb_monitor(self): """Callback to monitor the heartbeat from the controller""" self._hb_listener.flush() if self._hb_last_monitored > self._hb_last_pinged: self._hb_missed_beats += 1 self.log.warn( "No heartbeat in the last %s ms (%s time(s) in a row).", self.hb_check_period, self._hb_missed_beats) else: #self.log.debug("Heartbeat received (after missing %s beats).", self._hb_missed_beats) self._hb_missed_beats = 0 if self._hb_missed_beats >= self.max_heartbeat_misses: self.log.fatal( "Maximum number of heartbeats misses reached (%s times %s ms), shutting down.", self.max_heartbeat_misses, self.hb_check_period) self.session.send(self.registrar, "unregistration_request", content=dict(id=self.id)) self.loop.stop() self._hb_last_monitored = time.time() def start(self): dc = ioloop.DelayedCallback(self.register, 0, self.loop) dc.start() self._abort_dc = ioloop.DelayedCallback(self.abort, self.timeout * 1000, self.loop) self._abort_dc.start()
class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): name = u'ipython' description = usage.cl_usage crash_handler_class = IPAppCrashHandler examples = _examples flags = Dict(flags) aliases = Dict(aliases) classes = List() def _classes_default(self): """This has to be in a method, for TerminalIPythonApp to be available.""" return [ InteractiveShellApp, # ShellApp comes before TerminalApp, because self.__class__, # it will also affect subclasses (e.g. QtConsole) TerminalInteractiveShell, PromptManager, HistoryManager, ProfileDir, PlainTextFormatter, IPCompleter, ScriptMagics, StoreMagics, ] subcommands = dict( qtconsole=('IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', """Launch the IPython Qt Console."""), notebook=('IPython.html.notebookapp.NotebookApp', """Launch the IPython HTML Notebook Server."""), profile=("IPython.core.profileapp.ProfileApp", "Create and manage IPython profiles."), kernel=("IPython.kernel.zmq.kernelapp.IPKernelApp", "Start a kernel without an attached frontend."), console=('IPython.terminal.console.app.ZMQTerminalIPythonApp', """Launch the IPython terminal-based Console."""), locate=('IPython.terminal.ipapp.LocateIPythonApp', LocateIPythonApp.description), history=('IPython.core.historyapp.HistoryApp', "Manage the IPython history database."), nbconvert=('IPython.nbconvert.nbconvertapp.NbConvertApp', "Convert notebooks to/from other formats."), trust=( 'IPython.nbformat.sign.TrustNotebookApp', "Sign notebooks to trust their potentially unsafe contents at load." ), ) subcommands['install-nbextension'] = ( "IPython.html.nbextensions.NBExtensionApp", "Install IPython notebook extension files") # *do* autocreate requested profile, but don't create the config file. auto_create = Bool(True) # configurables ignore_old_config = Bool( False, config=True, help="Suppress warning messages about legacy config files") quick = Bool( False, config=True, help= """Start IPython quickly by skipping the loading of config files.""") def _quick_changed(self, name, old, new): if new: self.load_config_file = lambda *a, **kw: None self.ignore_old_config = True display_banner = Bool( True, config=True, help="Whether to display a banner upon starting IPython.") # if there is code of files to run from the cmd line, don't interact # unless the --i flag (App.force_interact) is true. force_interact = Bool( False, config=True, help="""If a command or file is given via the command-line, e.g. 'ipython foo.py', start an interactive shell after executing the file or command.""") def _force_interact_changed(self, name, old, new): if new: self.interact = True def _file_to_run_changed(self, name, old, new): if new: self.something_to_run = True if new and not self.force_interact: self.interact = False _code_to_run_changed = _file_to_run_changed _module_to_run_changed = _file_to_run_changed # internal, not-configurable interact = Bool(True) something_to_run = Bool(False) def parse_command_line(self, argv=None): """override to allow old '-pylab' flag with deprecation warning""" argv = sys.argv[1:] if argv is None else argv if '-pylab' in argv: # deprecated `-pylab` given, # warn and transform into current syntax argv = argv[:] # copy, don't clobber idx = argv.index('-pylab') warn.warn( "`-pylab` flag has been deprecated.\n" " Use `--matplotlib <backend>` and import pylab manually.") argv[idx] = '--pylab' return super(TerminalIPythonApp, self).parse_command_line(argv) @catch_config_error def initialize(self, argv=None): """Do actions after construct, but before starting the app.""" super(TerminalIPythonApp, self).initialize(argv) if self.subapp is not None: # don't bother initializing further, starting subapp return if not self.ignore_old_config: check_for_old_config(self.ipython_dir) # print self.extra_args if self.extra_args and not self.something_to_run: self.file_to_run = self.extra_args[0] self.init_path() # create the shell self.init_shell() # and draw the banner self.init_banner() # Now a variety of things that happen after the banner is printed. self.init_gui_pylab() self.init_extensions() self.init_code() def init_shell(self): """initialize the InteractiveShell instance""" # Create an InteractiveShell instance. # shell.display_banner should always be False for the terminal # based app, because we call shell.show_banner() by hand below # so the banner shows *before* all extension loading stuff. self.shell = TerminalInteractiveShell.instance( parent=self, display_banner=False, profile_dir=self.profile_dir, ipython_dir=self.ipython_dir, user_ns=self.user_ns) self.shell.configurables.append(self) def init_banner(self): """optionally display the banner""" if self.display_banner and self.interact: self.shell.show_banner() # Make sure there is a space below the banner. if self.log_level <= logging.INFO: print() def _pylab_changed(self, name, old, new): """Replace --pylab='inline' with --pylab='auto'""" if new == 'inline': warn.warn("'inline' not available as pylab backend, " "using 'auto' instead.") self.pylab = 'auto' def start(self): if self.subapp is not None: return self.subapp.start() # perform any prexec steps: if self.interact: self.log.debug("Starting IPython's mainloop...") self.shell.mainloop() else: self.log.debug("IPython not interactive...")
class BaseFormatter(Configurable): """A base formatter class that is configurable. This formatter should usually be used as the base class of all formatters. It is a traited :class:`Configurable` class and includes an extensible API for users to determine how their objects are formatted. The following logic is used to find a function to format an given object. 1. The object is introspected to see if it has a method with the name :attr:`print_method`. If is does, that object is passed to that method for formatting. 2. If no print method is found, three internal dictionaries are consulted to find print method: :attr:`singleton_printers`, :attr:`type_printers` and :attr:`deferred_printers`. Users should use these dictionaries to register functions that will be used to compute the format data for their objects (if those objects don't have the special print methods). The easiest way of using these dictionaries is through the :meth:`for_type` and :meth:`for_type_by_name` methods. If no function/callable is found to compute the format data, ``None`` is returned and this format type is not used. """ format_type = Unicode('text/plain') enabled = Bool(True, config=True) print_method = ObjectName('__repr__') # The singleton printers. # Maps the IDs of the builtin singleton objects to the format functions. singleton_printers = Dict(config=True) def _singleton_printers_default(self): return {} # The type-specific printers. # Map type objects to the format functions. type_printers = Dict(config=True) def _type_printers_default(self): return {} # The deferred-import type-specific printers. # Map (modulename, classname) pairs to the format functions. deferred_printers = Dict(config=True) def _deferred_printers_default(self): return {} def __call__(self, obj): """Compute the format for an object.""" if self.enabled: obj_id = id(obj) try: obj_class = getattr(obj, '__class__', None) or type(obj) # First try to find registered singleton printers for the type. try: printer = self.singleton_printers[obj_id] except (TypeError, KeyError): pass else: return printer(obj) # Next look for type_printers. for cls in pretty._get_mro(obj_class): if cls in self.type_printers: return self.type_printers[cls](obj) else: printer = self._in_deferred_types(cls) if printer is not None: return printer(obj) # Finally look for special method names. if hasattr(obj_class, self.print_method): printer = getattr(obj_class, self.print_method) return printer(obj) return None except Exception: pass else: return None def for_type(self, typ, func): """Add a format function for a given type. Parameters ----------- typ : class The class of the object that will be formatted using `func`. func : callable The callable that will be called to compute the format data. The call signature of this function is simple, it must take the object to be formatted and return the raw data for the given format. Subclasses may use a different call signature for the `func` argument. """ oldfunc = self.type_printers.get(typ, None) if func is not None: # To support easy restoration of old printers, we need to ignore # Nones. self.type_printers[typ] = func return oldfunc def for_type_by_name(self, type_module, type_name, func): """Add a format function for a type specified by the full dotted module and name of the type, rather than the type of the object. Parameters ---------- type_module : str The full dotted name of the module the type is defined in, like ``numpy``. type_name : str The name of the type (the class name), like ``dtype`` func : callable The callable that will be called to compute the format data. The call signature of this function is simple, it must take the object to be formatted and return the raw data for the given format. Subclasses may use a different call signature for the `func` argument. """ key = (type_module, type_name) oldfunc = self.deferred_printers.get(key, None) if func is not None: # To support easy restoration of old printers, we need to ignore # Nones. self.deferred_printers[key] = func return oldfunc def _in_deferred_types(self, cls): """ Check if the given class is specified in the deferred type registry. Returns the printer from the registry if it exists, and None if the class is not in the registry. Successful matches will be moved to the regular type registry for future use. """ mod = getattr(cls, '__module__', None) name = getattr(cls, '__name__', None) key = (mod, name) printer = None if key in self.deferred_printers: # Move the printer over to the regular registry. printer = self.deferred_printers.pop(key) self.type_printers[cls] = printer return printer
class IPControllerApp(BaseParallelApplication): name = u'ipcontroller' description = _description config_file_name = Unicode(default_config_file_name) classes = [ ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB ] + maybe_mongo # change default to True auto_create = Bool( True, config=True, help="""Whether to create profile dir if it doesn't exist.""") reuse_files = Bool(False, config=True, help='Whether to reuse existing json connection files.') secure = Bool( True, config=True, help='Whether to use HMAC digests for extra message authentication.') ssh_server = Unicode( u'', config=True, help="""ssh url for clients to use when connecting to the Controller processes. It should be of the form: [user@]server[:port]. The Controller's listening addresses must be accessible from the ssh server""", ) location = Unicode( u'', config=True, help= """The external IP or domain name of the Controller, used for disambiguating engine and client connections.""", ) import_statements = List( [], config=True, help= "import statements to be run at startup. Necessary in some environments" ) use_threads = Bool( False, config=True, help='Use threads instead of processes for the schedulers', ) # internal children = List() mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') def _use_threads_changed(self, name, old, new): self.mq_class = 'zmq.devices.%sMonitoredQueue' % ('Thread' if new else 'Process') aliases = Dict(aliases) flags = Dict(flags) def save_connection_dict(self, fname, cdict): """save a connection dict to json file.""" c = self.config url = cdict['url'] location = cdict['location'] if not location: try: proto, ip, port = split_url(url) except AssertionError: pass else: location = socket.gethostbyname_ex(socket.gethostname())[2][-1] cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) with open(fname, 'wb') as f: f.write(json.dumps(cdict, indent=2)) os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR) def load_config_from_json(self): """load config from existing json connector files.""" c = self.config # load from engine config with open( os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f: cfg = json.loads(f.read()) key = c.Session.key = asbytes(cfg['exec_key']) xport, addr = cfg['url'].split('://') c.HubFactory.engine_transport = xport ip, ports = addr.split(':') c.HubFactory.engine_ip = ip c.HubFactory.regport = int(ports) self.location = cfg['location'] # load client config with open( os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f: cfg = json.loads(f.read()) assert key == cfg[ 'exec_key'], "exec_key mismatch between engine and client keys" xport, addr = cfg['url'].split('://') c.HubFactory.client_transport = xport ip, ports = addr.split(':') c.HubFactory.client_ip = ip self.ssh_server = cfg['ssh'] assert int(ports) == c.HubFactory.regport, "regport mismatch" def init_hub(self): c = self.config self.do_import_statements() reusing = self.reuse_files if reusing: try: self.load_config_from_json() except (AssertionError, IOError): reusing = False # check again, because reusing may have failed: if reusing: pass elif self.secure: key = str(uuid.uuid4()) # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key) # with open(keyfile, 'w') as f: # f.write(key) # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR) c.Session.key = asbytes(key) else: key = c.Session.key = b'' try: self.factory = HubFactory(config=c, log=self.log) # self.start_logging() self.factory.init_hub() except: self.log.error("Couldn't construct the Controller", exc_info=True) self.exit(1) if not reusing: # save to new json config files f = self.factory cdict = { 'exec_key': key, 'ssh': self.ssh_server, 'url': "%s://%s:%s" % (f.client_transport, f.client_ip, f.regport), 'location': self.location } self.save_connection_dict('ipcontroller-client.json', cdict) edict = cdict edict['url'] = "%s://%s:%s" % ( (f.client_transport, f.client_ip, f.regport)) self.save_connection_dict('ipcontroller-engine.json', edict) # def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) hub = self.factory # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A', b'iopub') q.bind_in(hub.client_info['iopub']) q.bind_out(hub.engine_info['iopub']) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(hub.monitor_url) q.daemon = True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'in', b'out') q.bind_in(hub.client_info['mux']) q.setsockopt_in(zmq.IDENTITY, b'mux') q.bind_out(hub.engine_info['mux']) q.connect_mon(hub.monitor_url) q.daemon = True children.append(q) # Control Queue (in a Process) q = mq(zmq.XREP, zmq.XREP, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(hub.client_info['control']) q.setsockopt_in(zmq.IDENTITY, b'control') q.bind_out(hub.engine_info['control']) q.connect_mon(hub.monitor_url) q.daemon = True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure XREQ Task scheduler") q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(hub.client_info['task'][1]) q.setsockopt_in(zmq.IDENTITY, b'task') q.bind_out(hub.engine_info['task']) q.connect_mon(hub.monitor_url) q.daemon = True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler" % scheme) sargs = (hub.client_info['task'][1], hub.engine_info['task'], hub.monitor_url, hub.client_info['notification']) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url=self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon = True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs) def save_urls(self): """save the registration urls to files.""" c = self.config sec_dir = self.profile_dir.security_dir cf = self.factory with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f: f.write("%s://%s:%s" % (cf.engine_transport, cf.engine_ip, cf.regport)) with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f: f.write("%s://%s:%s" % (cf.client_transport, cf.client_ip, cf.regport)) def do_import_statements(self): statements = self.import_statements for s in statements: try: self.log.msg("Executing statement: '%s'" % s) exec s in globals(), locals() except: self.log.msg("Error running statement: %s" % s) def forward_logging(self): if self.log_url: self.log.info("Forwarding logging to %s" % self.log_url) context = zmq.Context.instance() lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = PUBHandler(lsock) self.log.removeHandler(self._log_handler) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler) self._log_handler = handler # # def initialize(self, argv=None): super(IPControllerApp, self).initialize(argv) self.forward_logging() self.init_hub() self.init_schedulers() def start(self): # Start the subprocesses: self.factory.start() child_procs = [] for child in self.children: child.start() if isinstance(child, ProcessMonitoredQueue): child_procs.append(child.launcher) elif isinstance(child, Process): child_procs.append(child) if child_procs: signal_children(child_procs) self.write_pid_file(overwrite=True) try: self.factory.loop.start() except KeyboardInterrupt: self.log.critical("Interrupted, Exiting...\n")
class VispyWidget(DOMWidget): _view_name = Unicode("VispyView", sync=True) _view_module = Unicode('/nbextensions/vispy/webgl-backend.js', sync=True) #height/width of the widget is managed by IPython. #it's a string and can be anything valid in CSS. #here we only manage the size of the viewport. width = Int(sync=True) height = Int(sync=True) resizable = Bool(value=True, sync=True) def __init__(self, **kwargs): super(VispyWidget, self).__init__(**kwargs) self.on_msg(self.events_received) self.canvas = None self.canvas_backend = None self.gen_event = None def set_canvas(self, canvas): self.width, self.height = canvas._backend._default_size self.canvas = canvas self.canvas_backend = self.canvas._backend self.canvas_backend.set_widget(self) self.gen_event = self.canvas_backend._gen_event #setup the backend widget then. # In IPython < 4, these callbacks are given two arguments; in # IPython/jupyter 4, they take 3. events_received is variadic to # accommodate both cases. def events_received(self, _, msg, *args): if msg['msg_type'] == 'init': self.canvas_backend._reinit_widget() elif msg['msg_type'] == 'events': events = msg['contents'] for ev in events: self.gen_event(ev) elif msg['msg_type'] == 'status': if msg['contents'] == 'removed': # Stop all timers associated to the widget. _stop_timers(self.canvas_backend._vispy_canvas) def send_glir_commands(self, commands): # TODO: check whether binary websocket is available (ipython >= 3) # Until IPython 3.0 is released, use base64. array_serialization = 'base64' # array_serialization = 'binary' if array_serialization == 'base64': msg = create_glir_message(commands, 'base64') msg['array_serialization'] = 'base64' self.send(msg) elif array_serialization == 'binary': msg = create_glir_message(commands, 'binary') msg['array_serialization'] = 'binary' # Remove the buffers from the JSON message: they will be sent # independently via binary WebSocket. buffers = msg.pop('buffers') self.comm.send({ "method": "custom", "content": msg }, buffers=buffers)
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' default_config_file_name='ipython_notebook_config.py' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager, FileNotebookManager] flags = Dict(flags) aliases = Dict(aliases) kernel_argv = List(Unicode) log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), default_value=logging.INFO, config=True, help="Set the log level by value or name.") # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('') # Network related information. ip = Unicode(LOCALHOST, config=True, help="The IP address the notebook server will listen on." ) def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on." ) port_retries = Integer(50, config=True, help="The number of additional ports to try if the specified port is not available." ) certfile = Unicode(u'', config=True, help="""The full path to an SSL/TLS certificate file.""" ) keyfile = Unicode(u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""" ) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """ ) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) read_only = Bool(False, config=True, help="Whether to prevent editing/execution of notebooks." ) webapp_settings = Dict(config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") enable_mathjax = Bool(True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """ ) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_project_url = Unicode('/', config=True, help='''The base URL for the notebook server''') base_kernel_url = Unicode('/', config=True, help='''The base URL for the kernel server''') websocket_host = Unicode("", config=True, help="""The hostname for the websocket server.""" ) extra_static_paths = List(Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""" ) def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [os.path.join(os.path.dirname(__file__), "static")] mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""" ) def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.webapp_settings.get("static_url_prefix", "/static/") try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), self.static_file_path) except IOError: if self.certfile: # HTTPS: load from Rackspace CDN, because SSL certificate requires it base = u"https://c328740.ssl.cf1.rackcdn.com" else: base = u"http://cdn.mathjax.org" url = base + u"/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url else: self.log.info("Using local MathJax from %s" % mathjax) return static_url_prefix+u"mathjax/MathJax.js" def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) notebook_manager_class = DottedObjectName('IPython.frontend.html.notebook.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if argv is None: argv = sys.argv[1:] # Scrub frontend-specific flags self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags) # Kernel should inherit default config file from frontend self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name) if self.extra_args: f = os.path.abspath(self.extra_args[0]) if os.path.isdir(f): nbdir = f else: self.file_to_run = f nbdir = os.path.dirname(f) self.config.NotebookManager.notebook_dir = nbdir def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_manager = MappingKernelManager( config=self.config, log=self.log, kernel_argv=self.kernel_argv, connection_dir = self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(config=self.config, log=self.log) self.notebook_manager.log_info() self.notebook_manager.load_notebook_names() self.cluster_manager = ClusterManager(config=self.config, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication( self, self.kernel_manager, self.notebook_manager, self.cluster_manager, self.log, self.base_project_url, self.webapp_settings ) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This" "is not recommended.") if not self.password and not self.read_only: self.log.critical(warning + "and not using authentication." "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries+1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno != errno.EADDRINUSE: raise self.log.info('The port %i is already in use, trying another random port.' % port) else: self.port = port success = True break if not success: self.log.critical('ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) def init_signal(self): # FIXME: remove this check when pyzmq dependency is >= 2.1.11 # safely extract zmq version info: try: zmq_v = zmq.pyzmq_version_info() except AttributeError: zmq_v = [ int(n) for n in re.findall(r'\d+', zmq.__version__) ] if 'dev' in zmq.__version__: zmq_v.append(999) zmq_v = tuple(zmq_v) if zmq_v >= (2,1,9) and not sys.platform.startswith('win'): # This won't work with 2.1.7 and # 2.1.9-10 will log ugly 'Interrupted system call' messages, # but it will work signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 time.sleep(0.1) sys.stdout.write("Shutdown Notebook Server (y/[n])? ") sys.stdout.flush() r,w,x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y'): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print "No answer for 5s:", print "resuming operation..." # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() @catch_config_error def initialize(self, argv=None): self.init_logging() super(NotebookApp, self).initialize(argv) self.init_configurables() self.init_webapp() self.init_signal() def cleanup_kernels(self): """shutdown all kernels The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') km = self.kernel_manager # copy list, since shutdown_kernel deletes keys for kid in list(km.kernel_ids): km.shutdown_kernel(kid) def start(self): ip = self.ip if self.ip else '[all ip addresses on your system]' proto = 'https' if self.certfile else 'http' info = self.log.info info("The IPython Notebook is running at: %s://%s:%i%s" % (proto, ip, self.port,self.base_project_url) ) info("Use Control-C to stop this server and shut down all kernels.") if self.open_browser or self.file_to_run: ip = self.ip or '127.0.0.1' try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: name, _ = os.path.splitext(os.path.basename(self.file_to_run)) url = self.notebook_manager.rev_mapping.get(name, '') else: url = '' if browser: b = lambda : browser.open("%s://%s:%i%s%s" % (proto, ip, self.port, self.base_project_url, url), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels()
class Client(HasTraits): """A semi-synchronous client to the IPython ZMQ cluster Parameters ---------- url_or_file : bytes; zmq url or path to ipcontroller-client.json Connection information for the Hub's registration. If a json connector file is given, then likely no further configuration is necessary. [Default: use profile] profile : bytes The name of the Cluster profile to be used to find connector information. [Default: 'default'] context : zmq.Context Pass an existing zmq.Context instance, otherwise the client will create its own. username : bytes set username to be passed to the Session object debug : bool flag for lots of message printing for debug purposes #-------------- ssh related args ---------------- # These are args for configuring the ssh tunnel to be used # credentials are used to forward connections over ssh to the Controller # Note that the ip given in `addr` needs to be relative to sshserver # The most basic case is to leave addr as pointing to localhost (127.0.0.1), # and set sshserver as the same machine the Controller is on. However, # the only requirement is that sshserver is able to see the Controller # (i.e. is within the same trusted network). sshserver : str A string of the form passed to ssh, i.e. 'server.tld' or '[email protected]:port' If keyfile or password is specified, and this is not, it will default to the ip given in addr. sshkey : str; path to public ssh key file This specifies a key to be used in ssh login, default None. Regular default ssh keys will be used without specifying this argument. password : str Your ssh password to sshserver. Note that if this is left None, you will be prompted for it if passwordless key based login is unavailable. paramiko : bool flag for whether to use paramiko instead of shell ssh for tunneling. [default: True on win32, False else] ------- exec authentication args ------- If even localhost is untrusted, you can have some protection against unauthorized execution by using a key. Messages are still sent as cleartext, so if someone can snoop your loopback traffic this will not help against malicious attacks. exec_key : str an authentication key or file containing a key default: None Attributes ---------- ids : list of int engine IDs requesting the ids attribute always synchronizes the registration state. To request ids without synchronization, use semi-private _ids attributes. history : list of msg_ids a list of msg_ids, keeping track of all the execution messages you have submitted in order. outstanding : set of msg_ids a set of msg_ids that have been submitted, but whose results have not yet been received. results : dict a dict of all our results, keyed by msg_id block : bool determines default behavior when block not specified in execution methods Methods ------- spin flushes incoming results and registration state changes control methods spin, and requesting `ids` also ensures up to date wait wait on one or more msg_ids execution methods apply legacy: execute, run data movement push, pull, scatter, gather query methods queue_status, get_result, purge, result_status control methods abort, shutdown """ block = Bool(False) outstanding = Set() results = Instance('collections.defaultdict', (dict, )) metadata = Instance('collections.defaultdict', (Metadata, )) history = List() debug = Bool(False) profile = CUnicode('default') _outstanding_dict = Instance('collections.defaultdict', (set, )) _ids = List() _connected = Bool(False) _ssh = Bool(False) _context = Instance('zmq.Context') _config = Dict() _engines = Instance(util.ReverseDict, (), {}) # _hub_socket=Instance('zmq.Socket') _query_socket = Instance('zmq.Socket') _control_socket = Instance('zmq.Socket') _iopub_socket = Instance('zmq.Socket') _notification_socket = Instance('zmq.Socket') _mux_socket = Instance('zmq.Socket') _task_socket = Instance('zmq.Socket') _task_scheme = Str() _closed = False _ignored_control_replies = Int(0) _ignored_hub_replies = Int(0) def __init__(self, url_or_file=None, profile='default', cluster_dir=None, ipython_dir=None, context=None, username=None, debug=False, exec_key=None, sshserver=None, sshkey=None, password=None, paramiko=None, timeout=10): super(Client, self).__init__(debug=debug, profile=profile) if context is None: context = zmq.Context.instance() self._context = context self._setup_cluster_dir(profile, cluster_dir, ipython_dir) if self._cd is not None: if url_or_file is None: url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json') assert url_or_file is not None, "I can't find enough information to connect to a hub!"\ " Please specify at least one of url_or_file or profile." try: util.validate_url(url_or_file) except AssertionError: if not os.path.exists(url_or_file): if self._cd: url_or_file = os.path.join(self._cd.security_dir, url_or_file) assert os.path.exists( url_or_file ), "Not a valid connection file or url: %r" % url_or_file with open(url_or_file) as f: cfg = json.loads(f.read()) else: cfg = {'url': url_or_file} # sync defaults from args, json: if sshserver: cfg['ssh'] = sshserver if exec_key: cfg['exec_key'] = exec_key exec_key = cfg['exec_key'] sshserver = cfg['ssh'] url = cfg['url'] location = cfg.setdefault('location', None) cfg['url'] = util.disambiguate_url(cfg['url'], location) url = cfg['url'] self._config = cfg self._ssh = bool(sshserver or sshkey or password) if self._ssh and sshserver is None: # default to ssh via localhost sshserver = url.split('://')[1].split(':')[0] if self._ssh and password is None: if tunnel.try_passwordless_ssh(sshserver, sshkey, paramiko): password = False else: password = getpass("SSH Password for %s: " % sshserver) ssh_kwargs = dict(keyfile=sshkey, password=password, paramiko=paramiko) if exec_key is not None and os.path.isfile(exec_key): arg = 'keyfile' else: arg = 'key' key_arg = {arg: exec_key} if username is None: self.session = ss.StreamSession(**key_arg) else: self.session = ss.StreamSession(username, **key_arg) self._query_socket = self._context.socket(zmq.XREQ) self._query_socket.setsockopt(zmq.IDENTITY, self.session.session) if self._ssh: tunnel.tunnel_connection(self._query_socket, url, sshserver, **ssh_kwargs) else: self._query_socket.connect(url) self.session.debug = self.debug self._notification_handlers = { 'registration_notification': self._register_engine, 'unregistration_notification': self._unregister_engine, 'shutdown_notification': lambda msg: self.close(), } self._queue_handlers = { 'execute_reply': self._handle_execute_reply, 'apply_reply': self._handle_apply_reply } self._connect(sshserver, ssh_kwargs, timeout) def __del__(self): """cleanup sockets, but _not_ context.""" self.close() def _setup_cluster_dir(self, profile, cluster_dir, ipython_dir): if ipython_dir is None: ipython_dir = get_ipython_dir() if cluster_dir is not None: try: self._cd = ClusterDir.find_cluster_dir(cluster_dir) return except ClusterDirError: pass elif profile is not None: try: self._cd = ClusterDir.find_cluster_dir_by_profile( ipython_dir, profile) return except ClusterDirError: pass self._cd = None def _update_engines(self, engines): """Update our engines dict and _ids from a dict of the form: {id:uuid}.""" for k, v in engines.iteritems(): eid = int(k) self._engines[eid] = bytes(v) # force not unicode self._ids.append(eid) self._ids = sorted(self._ids) if sorted(self._engines.keys()) != range(len(self._engines)) and \ self._task_scheme == 'pure' and self._task_socket: self._stop_scheduling_tasks() def _stop_scheduling_tasks(self): """Stop scheduling tasks because an engine has been unregistered from a pure ZMQ scheduler. """ self._task_socket.close() self._task_socket = None msg = "An engine has been unregistered, and we are using pure " +\ "ZMQ task scheduling. Task farming will be disabled." if self.outstanding: msg += " If you were running tasks when this happened, " +\ "some `outstanding` msg_ids may never resolve." warnings.warn(msg, RuntimeWarning) def _build_targets(self, targets): """Turn valid target IDs or 'all' into two lists: (int_ids, uuids). """ if not self._ids: # flush notification socket if no engines yet, just in case if not self.ids: raise error.NoEnginesRegistered( "Can't build targets without any engines") if targets is None: targets = self._ids elif isinstance(targets, str): if targets.lower() == 'all': targets = self._ids else: raise TypeError("%r not valid str target, must be 'all'" % (targets)) elif isinstance(targets, int): if targets < 0: targets = self.ids[targets] if targets not in self._ids: raise IndexError("No such engine: %i" % targets) targets = [targets] if isinstance(targets, slice): indices = range(len(self._ids))[targets] ids = self.ids targets = [ids[i] for i in indices] if not isinstance(targets, (tuple, list, xrange)): raise TypeError( "targets by int/slice/collection of ints only, not %s" % (type(targets))) return [self._engines[t] for t in targets], list(targets) def _connect(self, sshserver, ssh_kwargs, timeout): """setup all our socket connections to the cluster. This is called from __init__.""" # Maybe allow reconnecting? if self._connected: return self._connected = True def connect_socket(s, url): url = util.disambiguate_url(url, self._config['location']) if self._ssh: return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs) else: return s.connect(url) self.session.send(self._query_socket, 'connection_request') r, w, x = zmq.select([self._query_socket], [], [], timeout) if not r: raise error.TimeoutError("Hub connection request timed out") idents, msg = self.session.recv(self._query_socket, mode=0) if self.debug: pprint(msg) msg = ss.Message(msg) content = msg.content self._config['registration'] = dict(content) if content.status == 'ok': if content.mux: self._mux_socket = self._context.socket(zmq.XREQ) self._mux_socket.setsockopt(zmq.IDENTITY, self.session.session) connect_socket(self._mux_socket, content.mux) if content.task: self._task_scheme, task_addr = content.task self._task_socket = self._context.socket(zmq.XREQ) self._task_socket.setsockopt(zmq.IDENTITY, self.session.session) connect_socket(self._task_socket, task_addr) if content.notification: self._notification_socket = self._context.socket(zmq.SUB) connect_socket(self._notification_socket, content.notification) self._notification_socket.setsockopt(zmq.SUBSCRIBE, b'') # if content.query: # self._query_socket = self._context.socket(zmq.XREQ) # self._query_socket.setsockopt(zmq.IDENTITY, self.session.session) # connect_socket(self._query_socket, content.query) if content.control: self._control_socket = self._context.socket(zmq.XREQ) self._control_socket.setsockopt(zmq.IDENTITY, self.session.session) connect_socket(self._control_socket, content.control) if content.iopub: self._iopub_socket = self._context.socket(zmq.SUB) self._iopub_socket.setsockopt(zmq.SUBSCRIBE, b'') self._iopub_socket.setsockopt(zmq.IDENTITY, self.session.session) connect_socket(self._iopub_socket, content.iopub) self._update_engines(dict(content.engines)) else: self._connected = False raise Exception("Failed to connect!") #-------------------------------------------------------------------------- # handlers and callbacks for incoming messages #-------------------------------------------------------------------------- def _unwrap_exception(self, content): """unwrap exception, and remap engine_id to int.""" e = error.unwrap_exception(content) # print e.traceback if e.engine_info: e_uuid = e.engine_info['engine_uuid'] eid = self._engines[e_uuid] e.engine_info['engine_id'] = eid return e def _extract_metadata(self, header, parent, content): md = { 'msg_id': parent['msg_id'], 'received': datetime.now(), 'engine_uuid': header.get('engine', None), 'follow': parent.get('follow', []), 'after': parent.get('after', []), 'status': content['status'], } if md['engine_uuid'] is not None: md['engine_id'] = self._engines.get(md['engine_uuid'], None) if 'date' in parent: md['submitted'] = datetime.strptime(parent['date'], util.ISO8601) if 'started' in header: md['started'] = datetime.strptime(header['started'], util.ISO8601) if 'date' in header: md['completed'] = datetime.strptime(header['date'], util.ISO8601) return md def _register_engine(self, msg): """Register a new engine, and update our connection info.""" content = msg['content'] eid = content['id'] d = {eid: content['queue']} self._update_engines(d) def _unregister_engine(self, msg): """Unregister an engine that has died.""" content = msg['content'] eid = int(content['id']) if eid in self._ids: self._ids.remove(eid) uuid = self._engines.pop(eid) self._handle_stranded_msgs(eid, uuid) if self._task_socket and self._task_scheme == 'pure': self._stop_scheduling_tasks() def _handle_stranded_msgs(self, eid, uuid): """Handle messages known to be on an engine when the engine unregisters. It is possible that this will fire prematurely - that is, an engine will go down after completing a result, and the client will be notified of the unregistration and later receive the successful result. """ outstanding = self._outstanding_dict[uuid] for msg_id in list(outstanding): if msg_id in self.results: # we already continue try: raise error.EngineError( "Engine %r died while running task %r" % (eid, msg_id)) except: content = error.wrap_exception() # build a fake message: parent = {} header = {} parent['msg_id'] = msg_id header['engine'] = uuid header['date'] = datetime.now().strftime(util.ISO8601) msg = dict(parent_header=parent, header=header, content=content) self._handle_apply_reply(msg) def _handle_execute_reply(self, msg): """Save the reply to an execute_request into our results. execute messages are never actually used. apply is used instead. """ parent = msg['parent_header'] msg_id = parent['msg_id'] if msg_id not in self.outstanding: if msg_id in self.history: print("got stale result: %s" % msg_id) else: print("got unknown result: %s" % msg_id) else: self.outstanding.remove(msg_id) self.results[msg_id] = self._unwrap_exception(msg['content']) def _handle_apply_reply(self, msg): """Save the reply to an apply_request into our results.""" parent = msg['parent_header'] msg_id = parent['msg_id'] if msg_id not in self.outstanding: if msg_id in self.history: print("got stale result: %s" % msg_id) print self.results[msg_id] print msg else: print("got unknown result: %s" % msg_id) else: self.outstanding.remove(msg_id) content = msg['content'] header = msg['header'] # construct metadata: md = self.metadata[msg_id] md.update(self._extract_metadata(header, parent, content)) # is this redundant? self.metadata[msg_id] = md e_outstanding = self._outstanding_dict[md['engine_uuid']] if msg_id in e_outstanding: e_outstanding.remove(msg_id) # construct result: if content['status'] == 'ok': self.results[msg_id] = util.unserialize_object(msg['buffers'])[0] elif content['status'] == 'aborted': self.results[msg_id] = error.TaskAborted(msg_id) elif content['status'] == 'resubmitted': # TODO: handle resubmission pass else: self.results[msg_id] = self._unwrap_exception(content) def _flush_notifications(self): """Flush notifications of engine registrations waiting in ZMQ queue.""" msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) while msg is not None: if self.debug: pprint(msg) msg = msg[-1] msg_type = msg['msg_type'] handler = self._notification_handlers.get(msg_type, None) if handler is None: raise Exception("Unhandled message type: %s" % msg.msg_type) else: handler(msg) msg = self.session.recv(self._notification_socket, mode=zmq.NOBLOCK) def _flush_results(self, sock): """Flush task or queue results waiting in ZMQ queue.""" msg = self.session.recv(sock, mode=zmq.NOBLOCK) while msg is not None: if self.debug: pprint(msg) msg = msg[-1] msg_type = msg['msg_type'] handler = self._queue_handlers.get(msg_type, None) if handler is None: raise Exception("Unhandled message type: %s" % msg.msg_type) else: handler(msg) msg = self.session.recv(sock, mode=zmq.NOBLOCK) def _flush_control(self, sock): """Flush replies from the control channel waiting in the ZMQ queue. Currently: ignore them.""" if self._ignored_control_replies <= 0: return msg = self.session.recv(sock, mode=zmq.NOBLOCK) while msg is not None: self._ignored_control_replies -= 1 if self.debug: pprint(msg) msg = self.session.recv(sock, mode=zmq.NOBLOCK) def _flush_ignored_control(self): """flush ignored control replies""" while self._ignored_control_replies > 0: self.session.recv(self._control_socket) self._ignored_control_replies -= 1 def _flush_ignored_hub_replies(self): msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) while msg is not None: msg = self.session.recv(self._query_socket, mode=zmq.NOBLOCK) def _flush_iopub(self, sock): """Flush replies from the iopub channel waiting in the ZMQ queue. """ msg = self.session.recv(sock, mode=zmq.NOBLOCK) while msg is not None: if self.debug: pprint(msg) msg = msg[-1] parent = msg['parent_header'] msg_id = parent['msg_id'] content = msg['content'] header = msg['header'] msg_type = msg['msg_type'] # init metadata: md = self.metadata[msg_id] if msg_type == 'stream': name = content['name'] s = md[name] or '' md[name] = s + content['data'] elif msg_type == 'pyerr': md.update({'pyerr': self._unwrap_exception(content)}) elif msg_type == 'pyin': md.update({'pyin': content['code']}) else: md.update({msg_type: content.get('data', '')}) # reduntant? self.metadata[msg_id] = md msg = self.session.recv(sock, mode=zmq.NOBLOCK) #-------------------------------------------------------------------------- # len, getitem #-------------------------------------------------------------------------- def __len__(self): """len(client) returns # of engines.""" return len(self.ids) def __getitem__(self, key): """index access returns DirectView multiplexer objects Must be int, slice, or list/tuple/xrange of ints""" if not isinstance(key, (int, slice, tuple, list, xrange)): raise TypeError("key by int/slice/iterable of ints only, not %s" % (type(key))) else: return self.direct_view(key) #-------------------------------------------------------------------------- # Begin public methods #-------------------------------------------------------------------------- @property def ids(self): """Always up-to-date ids property.""" self._flush_notifications() # always copy: return list(self._ids) def close(self): if self._closed: return snames = filter(lambda n: n.endswith('socket'), dir(self)) for socket in map(lambda name: getattr(self, name), snames): if isinstance(socket, zmq.Socket) and not socket.closed: socket.close() self._closed = True def spin(self): """Flush any registration notifications and execution results waiting in the ZMQ queue. """ if self._notification_socket: self._flush_notifications() if self._mux_socket: self._flush_results(self._mux_socket) if self._task_socket: self._flush_results(self._task_socket) if self._control_socket: self._flush_control(self._control_socket) if self._iopub_socket: self._flush_iopub(self._iopub_socket) if self._query_socket: self._flush_ignored_hub_replies() def wait(self, jobs=None, timeout=-1): """waits on one or more `jobs`, for up to `timeout` seconds. Parameters ---------- jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects ints are indices to self.history strs are msg_ids default: wait on all outstanding messages timeout : float a time in seconds, after which to give up. default is -1, which means no timeout Returns ------- True : when all msg_ids are done False : timeout reached, some msg_ids still outstanding """ tic = time.time() if jobs is None: theids = self.outstanding else: if isinstance(jobs, (int, str, AsyncResult)): jobs = [jobs] theids = set() for job in jobs: if isinstance(job, int): # index access job = self.history[job] elif isinstance(job, AsyncResult): map(theids.add, job.msg_ids) continue theids.add(job) if not theids.intersection(self.outstanding): return True self.spin() while theids.intersection(self.outstanding): if timeout >= 0 and (time.time() - tic) > timeout: break time.sleep(1e-3) self.spin() return len(theids.intersection(self.outstanding)) == 0 #-------------------------------------------------------------------------- # Control methods #-------------------------------------------------------------------------- @spin_first def clear(self, targets=None, block=None): """Clear the namespace in target(s).""" block = self.block if block is None else block targets = self._build_targets(targets)[0] for t in targets: self.session.send(self._control_socket, 'clear_request', content={}, ident=t) error = False if block: self._flush_ignored_control() for i in range(len(targets)): idents, msg = self.session.recv(self._control_socket, 0) if self.debug: pprint(msg) if msg['content']['status'] != 'ok': error = self._unwrap_exception(msg['content']) else: self._ignored_control_replies += len(targets) if error: raise error @spin_first def abort(self, jobs=None, targets=None, block=None): """Abort specific jobs from the execution queues of target(s). This is a mechanism to prevent jobs that have already been submitted from executing. Parameters ---------- jobs : msg_id, list of msg_ids, or AsyncResult The jobs to be aborted """ block = self.block if block is None else block targets = self._build_targets(targets)[0] msg_ids = [] if isinstance(jobs, (basestring, AsyncResult)): jobs = [jobs] bad_ids = filter( lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs) if bad_ids: raise TypeError( "Invalid msg_id type %r, expected str or AsyncResult" % bad_ids[0]) for j in jobs: if isinstance(j, AsyncResult): msg_ids.extend(j.msg_ids) else: msg_ids.append(j) content = dict(msg_ids=msg_ids) for t in targets: self.session.send(self._control_socket, 'abort_request', content=content, ident=t) error = False if block: self._flush_ignored_control() for i in range(len(targets)): idents, msg = self.session.recv(self._control_socket, 0) if self.debug: pprint(msg) if msg['content']['status'] != 'ok': error = self._unwrap_exception(msg['content']) else: self._ignored_control_replies += len(targets) if error: raise error @spin_first def shutdown(self, targets=None, restart=False, hub=False, block=None): """Terminates one or more engine processes, optionally including the hub.""" block = self.block if block is None else block if hub: targets = 'all' targets = self._build_targets(targets)[0] for t in targets: self.session.send(self._control_socket, 'shutdown_request', content={'restart': restart}, ident=t) error = False if block or hub: self._flush_ignored_control() for i in range(len(targets)): idents, msg = self.session.recv(self._control_socket, 0) if self.debug: pprint(msg) if msg['content']['status'] != 'ok': error = self._unwrap_exception(msg['content']) else: self._ignored_control_replies += len(targets) if hub: time.sleep(0.25) self.session.send(self._query_socket, 'shutdown_request') idents, msg = self.session.recv(self._query_socket, 0) if self.debug: pprint(msg) if msg['content']['status'] != 'ok': error = self._unwrap_exception(msg['content']) if error: raise error #-------------------------------------------------------------------------- # Execution related methods #-------------------------------------------------------------------------- def _maybe_raise(self, result): """wrapper for maybe raising an exception if apply failed.""" if isinstance(result, error.RemoteError): raise result return result def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, track=False, ident=None): """construct and send an apply message via a socket. This is the principal method with which all engine execution is performed by views. """ assert not self._closed, "cannot use me anymore, I'm closed!" # defaults: args = args if args is not None else [] kwargs = kwargs if kwargs is not None else {} subheader = subheader if subheader is not None else {} # validate arguments if not callable(f): raise TypeError("f must be callable, not %s" % type(f)) if not isinstance(args, (tuple, list)): raise TypeError("args must be tuple or list, not %s" % type(args)) if not isinstance(kwargs, dict): raise TypeError("kwargs must be dict, not %s" % type(kwargs)) if not isinstance(subheader, dict): raise TypeError("subheader must be dict, not %s" % type(subheader)) bufs = util.pack_apply_message(f, args, kwargs) msg = self.session.send(socket, "apply_request", buffers=bufs, ident=ident, subheader=subheader, track=track) msg_id = msg['msg_id'] self.outstanding.add(msg_id) if ident: # possibly routed to a specific engine if isinstance(ident, list): ident = ident[-1] if ident in self._engines.values(): # save for later, in case of engine death self._outstanding_dict[ident].add(msg_id) self.history.append(msg_id) self.metadata[msg_id]['submitted'] = datetime.now() return msg #-------------------------------------------------------------------------- # construct a View object #-------------------------------------------------------------------------- def load_balanced_view(self, targets=None): """construct a DirectView object. If no arguments are specified, create a LoadBalancedView using all engines. Parameters ---------- targets: list,slice,int,etc. [default: use all engines] The subset of engines across which to load-balance """ if targets is not None: targets = self._build_targets(targets)[1] return LoadBalancedView(client=self, socket=self._task_socket, targets=targets) def direct_view(self, targets='all'): """construct a DirectView object. If no targets are specified, create a DirectView using all engines. Parameters ---------- targets: list,slice,int,etc. [default: use all engines] The engines to use for the View """ single = isinstance(targets, int) targets = self._build_targets(targets)[1] if single: targets = targets[0] return DirectView(client=self, socket=self._mux_socket, targets=targets) #-------------------------------------------------------------------------- # Query methods #-------------------------------------------------------------------------- @spin_first def get_result(self, indices_or_msg_ids=None, block=None): """Retrieve a result by msg_id or history index, wrapped in an AsyncResult object. If the client already has the results, no request to the Hub will be made. This is a convenient way to construct AsyncResult objects, which are wrappers that include metadata about execution, and allow for awaiting results that were not submitted by this Client. It can also be a convenient way to retrieve the metadata associated with blocking execution, since it always retrieves Examples -------- :: In [10]: r = client.apply() Parameters ---------- indices_or_msg_ids : integer history index, str msg_id, or list of either The indices or msg_ids of indices to be retrieved block : bool Whether to wait for the result to be done Returns ------- AsyncResult A single AsyncResult object will always be returned. AsyncHubResult A subclass of AsyncResult that retrieves results from the Hub """ block = self.block if block is None else block if indices_or_msg_ids is None: indices_or_msg_ids = -1 if not isinstance(indices_or_msg_ids, (list, tuple)): indices_or_msg_ids = [indices_or_msg_ids] theids = [] for id in indices_or_msg_ids: if isinstance(id, int): id = self.history[id] if not isinstance(id, str): raise TypeError("indices must be str or int, not %r" % id) theids.append(id) local_ids = filter( lambda msg_id: msg_id in self.history or msg_id in self.results, theids) remote_ids = filter(lambda msg_id: msg_id not in local_ids, theids) if remote_ids: ar = AsyncHubResult(self, msg_ids=theids) else: ar = AsyncResult(self, msg_ids=theids) if block: ar.wait() return ar @spin_first def result_status(self, msg_ids, status_only=True): """Check on the status of the result(s) of the apply request with `msg_ids`. If status_only is False, then the actual results will be retrieved, else only the status of the results will be checked. Parameters ---------- msg_ids : list of msg_ids if int: Passed as index to self.history for convenience. status_only : bool (default: True) if False: Retrieve the actual results of completed tasks. Returns ------- results : dict There will always be the keys 'pending' and 'completed', which will be lists of msg_ids that are incomplete or complete. If `status_only` is False, then completed results will be keyed by their `msg_id`. """ if not isinstance(msg_ids, (list, tuple)): msg_ids = [msg_ids] theids = [] for msg_id in msg_ids: if isinstance(msg_id, int): msg_id = self.history[msg_id] if not isinstance(msg_id, basestring): raise TypeError("msg_ids must be str, not %r" % msg_id) theids.append(msg_id) completed = [] local_results = {} # comment this block out to temporarily disable local shortcut: for msg_id in theids: if msg_id in self.results: completed.append(msg_id) local_results[msg_id] = self.results[msg_id] theids.remove(msg_id) if theids: # some not locally cached content = dict(msg_ids=theids, status_only=status_only) msg = self.session.send(self._query_socket, "result_request", content=content) zmq.select([self._query_socket], [], []) idents, msg = self.session.recv(self._query_socket, zmq.NOBLOCK) if self.debug: pprint(msg) content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) buffers = msg['buffers'] else: content = dict(completed=[], pending=[]) content['completed'].extend(completed) if status_only: return content failures = [] # load cached results into result: content.update(local_results) # update cache with results: for msg_id in sorted(theids): if msg_id in content['completed']: rec = content[msg_id] parent = rec['header'] header = rec['result_header'] rcontent = rec['result_content'] iodict = rec['io'] if isinstance(rcontent, str): rcontent = self.session.unpack(rcontent) md = self.metadata[msg_id] md.update(self._extract_metadata(header, parent, rcontent)) md.update(iodict) if rcontent['status'] == 'ok': res, buffers = util.unserialize_object(buffers) else: print rcontent res = self._unwrap_exception(rcontent) failures.append(res) self.results[msg_id] = res content[msg_id] = res if len(theids) == 1 and failures: raise failures[0] error.collect_exceptions(failures, "result_status") return content @spin_first def queue_status(self, targets='all', verbose=False): """Fetch the status of engine queues. Parameters ---------- targets : int/str/list of ints/strs the engines whose states are to be queried. default : all verbose : bool Whether to return lengths only, or lists of ids for each element """ engine_ids = self._build_targets(targets)[1] content = dict(targets=engine_ids, verbose=verbose) self.session.send(self._query_socket, "queue_request", content=content) idents, msg = self.session.recv(self._query_socket, 0) if self.debug: pprint(msg) content = msg['content'] status = content.pop('status') if status != 'ok': raise self._unwrap_exception(content) content = util.rekey(content) if isinstance(targets, int): return content[targets] else: return content @spin_first def purge_results(self, jobs=[], targets=[]): """Tell the Hub to forget results. Individual results can be purged by msg_id, or the entire history of specific targets can be purged. Parameters ---------- jobs : str or list of str or AsyncResult objects the msg_ids whose results should be forgotten. targets : int/str/list of ints/strs The targets, by uuid or int_id, whose entire history is to be purged. Use `targets='all'` to scrub everything from the Hub's memory. default : None """ if not targets and not jobs: raise ValueError( "Must specify at least one of `targets` and `jobs`") if targets: targets = self._build_targets(targets)[1] # construct msg_ids from jobs msg_ids = [] if isinstance(jobs, (basestring, AsyncResult)): jobs = [jobs] bad_ids = filter( lambda obj: not isinstance(obj, (basestring, AsyncResult)), jobs) if bad_ids: raise TypeError( "Invalid msg_id type %r, expected str or AsyncResult" % bad_ids[0]) for j in jobs: if isinstance(j, AsyncResult): msg_ids.extend(j.msg_ids) else: msg_ids.append(j) content = dict(targets=targets, msg_ids=msg_ids) self.session.send(self._query_socket, "purge_request", content=content) idents, msg = self.session.recv(self._query_socket, 0) if self.debug: pprint(msg) content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) @spin_first def hub_history(self): """Get the Hub's history Just like the Client, the Hub has a history, which is a list of msg_ids. This will contain the history of all clients, and, depending on configuration, may contain history across multiple cluster sessions. Any msg_id returned here is a valid argument to `get_result`. Returns ------- msg_ids : list of strs list of all msg_ids, ordered by task submission time. """ self.session.send(self._query_socket, "history_request", content={}) idents, msg = self.session.recv(self._query_socket, 0) if self.debug: pprint(msg) content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) else: return content['history'] @spin_first def db_query(self, query, keys=None): """Query the Hub's TaskRecord database This will return a list of task record dicts that match `query` Parameters ---------- query : mongodb query dict The search dict. See mongodb query docs for details. keys : list of strs [optional] THe subset of keys to be returned. The default is to fetch everything. 'msg_id' will *always* be included. """ content = dict(query=query, keys=keys) self.session.send(self._query_socket, "db_request", content=content) idents, msg = self.session.recv(self._query_socket, 0) if self.debug: pprint(msg) content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) records = content['records'] buffer_lens = content['buffer_lens'] result_buffer_lens = content['result_buffer_lens'] buffers = msg['buffers'] has_bufs = buffer_lens is not None has_rbufs = result_buffer_lens is not None for i, rec in enumerate(records): # relink buffers if has_bufs: blen = buffer_lens[i] rec['buffers'], buffers = buffers[:blen], buffers[blen:] if has_rbufs: blen = result_buffer_lens[i] rec['result_buffers'], buffers = buffers[:blen], buffers[blen:] # turn timestamps back into times for key in 'submitted started completed resubmitted'.split(): maybedate = rec.get(key, None) if maybedate and util.ISO8601_RE.match(maybedate): rec[key] = datetime.strptime(maybedate, util.ISO8601) return records
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples classes = IPythonConsoleApp.classes + [ MappingKernelManager, NotebookManager, FileNotebookManager ] flags = Dict(flags) aliases = Dict(aliases) subcommands = dict(list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), ) kernel_argv = List(Unicode) def _log_level_default(self): return logging.INFO def _log_format_default(self): """override default log format to include time""" return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('') # Network related information. ip = Unicode(config=True, help="The IP address the notebook server will listen on.") def _ip_default(self): return localhost() def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on.") port_retries = Integer( 50, config=True, help= "The number of additional ports to try if the specified port is not available." ) certfile = Unicode( u'', config=True, help="""The full path to an SSL/TLS certificate file.""") keyfile = Unicode( u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""") cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """) def _cookie_secret_default(self): return os.urandom(1024) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict( config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") enable_mathjax = Bool( True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_url_changed(self, name, old, new): if not new.startswith('/'): self.base_url = '/' + new elif not new.endswith('/'): self.base_url = new + '/' base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""") def _base_project_url_changed(self, name, old, new): self.log.warn("base_project_url is deprecated, use base_url") self.base_url = new websocket_url = Unicode("", config=True, help="""The base URL for the websocket server, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """) extra_static_paths = List( Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""") def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] nbextensions_path = List( Unicode, config=True, help= """paths for Javascript extensions. By default, this is just IPYTHONDIR/nbextensions""" ) def _nbextensions_path_default(self): return [os.path.join(get_ipython_dir(), 'nbextensions')] mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""") def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.webapp_settings.get( "static_url_prefix", url_path_join(self.base_url, "static")) # try local mathjax, either in nbextensions/mathjax or static/mathjax for (url_prefix, search_path) in [ (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path), (static_url_prefix, self.static_file_path), ]: self.log.debug("searching for local mathjax in %s", search_path) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path) except IOError: continue else: url = url_path_join(url_prefix, u"mathjax/MathJax.js") self.log.info("Serving local MathJax from %s at %s", mathjax, url) return url # no local mathjax, serve from CDN if self.certfile: # HTTPS: load from Rackspace CDN, because SSL certificate requires it host = u"https://c328740.ssl.cf1.rackcdn.com" else: host = u"http://cdn.mathjax.org" url = host + u"/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) notebook_manager_class = DottedObjectName( 'IPython.html.services.notebooks.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') trust_xheaders = Bool( False, config=True, help= ("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL" )) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json" % os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: arg0 = self.extra_args[0] f = os.path.abspath(arg0) self.argv.remove(arg0) if not os.path.exists(f): self.log.critical("No such file or directory: %s", f) self.exit(1) if os.path.isdir(f): self.config.FileNotebookManager.notebook_dir = f elif os.path.isfile(f): self.file_to_run = f def init_kernel_argv(self): """construct the kernel arguments""" # Scrub frontend-specific flags self.kernel_argv = swallow_argv(self.argv, notebook_aliases, notebook_flags) if any(arg.startswith(u'--pylab') for arg in self.kernel_argv): self.log.warn('\n '.join([ "Starting all kernels in pylab mode is not recommended,", "and will be disabled in a future release.", "Please use the %matplotlib magic to enable matplotlib instead.", "pylab implies many imports, which can have confusing side effects", "and harm the reproducibility of your notebooks.", ])) # Kernel should inherit default config file from frontend self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) # Kernel should get *absolute* path to profile directory self.kernel_argv.extend(["--profile-dir", self.profile_dir.location]) def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_manager = MappingKernelManager( parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir=self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(parent=self, log=self.log) self.session_manager = SessionManager(parent=self, log=self.log) self.cluster_manager = ClusterManager(parent=self, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False # hook up tornado 3's loggers to our app handlers for name in ('access', 'application', 'general'): logger = logging.getLogger('tornado.%s' % name) logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication(self, self.kernel_manager, self.notebook_manager, self.cluster_manager, self.session_manager, self.log, self.base_url, self.webapp_settings) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical( warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries + 1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info( 'The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical( 'ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else localhost() return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 time.sleep(0.1) info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r, w, x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y'): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn( "components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn( "components submodule unclean, you may see 404s on static/components" ) self.log.warn( "run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.notebook_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return { 'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), 'base_url': self.base_url, 'notebook_dir': os.path.abspath(self.notebook_manager.notebook_dir), } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None f = self.file_to_run if f: nbdir = os.path.abspath(self.notebook_manager.notebook_dir) if f.startswith(nbdir): f = f[len(nbdir):] else: self.log.warn( "Probably won't be able to open notebook %s " "because it is not in notebook_dir %s", f, nbdir, ) if os.path.isfile(self.file_to_run): url = url_path_join('notebooks', f) else: url = url_path_join('tree', f) if browser: b = lambda: browser.open("%s%s" % (self.connection_url, url), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file()
class ProfileDir(LoggingConfigurable): """An object to manage the profile directory and its resources. The profile directory is used by all IPython applications, to manage configuration, logging and security. This object knows how to find, create and manage these directories. This should be used by any code that wants to handle profiles. """ security_dir_name = Unicode('security') log_dir_name = Unicode('log') startup_dir_name = Unicode('startup') pid_dir_name = Unicode('pid') static_dir_name = Unicode('static') security_dir = Unicode(u'') log_dir = Unicode(u'') startup_dir = Unicode(u'') pid_dir = Unicode(u'') static_dir = Unicode(u'') location = Unicode(u'', config=True, help="""Set the profile location directly. This overrides the logic used by the `profile` option.""", ) _location_isset = Bool(False) # flag for detecting multiply set location def _location_changed(self, name, old, new): if self._location_isset: raise RuntimeError("Cannot set profile location more than once.") self._location_isset = True num_tries = 0 max_tries = 5 while not os.path.isdir(new): try: os.makedirs(new) except OSError: if num_tries > max_tries: raise num_tries += 1 time.sleep(0.5) # ensure config files exist: self.security_dir = os.path.join(new, self.security_dir_name) self.log_dir = os.path.join(new, self.log_dir_name) self.startup_dir = os.path.join(new, self.startup_dir_name) self.pid_dir = os.path.join(new, self.pid_dir_name) self.static_dir = os.path.join(new, self.static_dir_name) self.check_dirs() def _log_dir_changed(self, name, old, new): self.check_log_dir() def _mkdir(self, path, mode=None): """ensure a directory exists at a given path This is a version of os.mkdir, with the following differences: - returns True if it created the directory, False otherwise - ignores EEXIST, protecting against race conditions where the dir may have been created in between the check and the creation - sets permissions if requested and the dir already exists """ if os.path.exists(path): if mode and os.stat(path).st_mode != mode: try: os.chmod(path, mode) except OSError: self.log.warn( "Could not set permissions on %s", path ) return False try: if mode: os.mkdir(path, mode) else: os.mkdir(path) except OSError as e: if e.errno == errno.EEXIST: return False else: raise return True def check_log_dir(self): self._mkdir(self.log_dir) def _startup_dir_changed(self, name, old, new): self.check_startup_dir() def check_startup_dir(self): self._mkdir(self.startup_dir) readme = os.path.join(self.startup_dir, 'README') src = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'README_STARTUP') if not os.path.exists(src): self.log.warn("Could not copy README_STARTUP to startup dir. Source file %s does not exist.", src) if os.path.exists(src) and not os.path.exists(readme): shutil.copy(src, readme) def _security_dir_changed(self, name, old, new): self.check_security_dir() def check_security_dir(self): self._mkdir(self.security_dir, 0o40700) def _pid_dir_changed(self, name, old, new): self.check_pid_dir() def check_pid_dir(self): self._mkdir(self.pid_dir, 0o40700) def _static_dir_changed(self, name, old, new): self.check_startup_dir() def check_static_dir(self): self._mkdir(self.static_dir) custom = os.path.join(self.static_dir, 'custom') self._mkdir(custom) from IPython.html import DEFAULT_STATIC_FILES_PATH for fname in ('custom.js', 'custom.css'): src = os.path.join(DEFAULT_STATIC_FILES_PATH, 'custom', fname) dest = os.path.join(custom, fname) if not os.path.exists(src): self.log.warn("Could not copy default file to static dir. Source file %s does not exist.", src) continue if not os.path.exists(dest): shutil.copy(src, dest) def check_dirs(self): self.check_security_dir() self.check_log_dir() self.check_pid_dir() self.check_startup_dir() self.check_static_dir() def copy_config_file(self, config_file, path=None, overwrite=False): """Copy a default config file into the active profile directory. Default configuration files are kept in :mod:`IPython.config.default`. This function moves these from that location to the working profile directory. """ dst = os.path.join(self.location, config_file) if os.path.isfile(dst) and not overwrite: return False if path is None: path = os.path.join(get_ipython_package_dir(), u'config', u'profile', u'default') src = os.path.join(path, config_file) shutil.copy(src, dst) return True @classmethod def create_profile_dir(cls, profile_dir, config=None): """Create a new profile directory given a full path. Parameters ---------- profile_dir : str The full path to the profile directory. If it does exist, it will be used. If not, it will be created. """ return cls(location=profile_dir, config=config) @classmethod def create_profile_dir_by_name(cls, path, name=u'default', config=None): """Create a profile dir by profile name and path. Parameters ---------- path : unicode The path (directory) to put the profile directory in. name : unicode The name of the profile. The name of the profile directory will be "profile_<profile>". """ if not os.path.isdir(path): raise ProfileDirError('Directory not found: %s' % path) profile_dir = os.path.join(path, u'profile_' + name) return cls(location=profile_dir, config=config) @classmethod def find_profile_dir_by_name(cls, ipython_dir, name=u'default', config=None): """Find an existing profile dir by profile name, return its ProfileDir. This searches through a sequence of paths for a profile dir. If it is not found, a :class:`ProfileDirError` exception will be raised. The search path algorithm is: 1. ``py3compat.getcwd()`` 2. ``ipython_dir`` Parameters ---------- ipython_dir : unicode or str The IPython directory to use. name : unicode or str The name of the profile. The name of the profile directory will be "profile_<profile>". """ dirname = u'profile_' + name paths = [py3compat.getcwd(), ipython_dir] for p in paths: profile_dir = os.path.join(p, dirname) if os.path.isdir(profile_dir): return cls(location=profile_dir, config=config) else: raise ProfileDirError('Profile directory not found in paths: %s' % dirname) @classmethod def find_profile_dir(cls, profile_dir, config=None): """Find/create a profile dir and return its ProfileDir. This will create the profile directory if it doesn't exist. Parameters ---------- profile_dir : unicode or str The path of the profile directory. This is expanded using :func:`IPython.utils.genutils.expand_path`. """ profile_dir = expand_path(profile_dir) if not os.path.isdir(profile_dir): raise ProfileDirError('Profile directory not found: %s' % profile_dir) return cls(location=profile_dir, config=config)
class PlainTextFormatter(BaseFormatter): """The default pretty-printer. This uses :mod:`IPython.lib.pretty` to compute the format data of the object. If the object cannot be pretty printed, :func:`repr` is used. See the documentation of :mod:`IPython.lib.pretty` for details on how to write pretty printers. Here is a simple example:: def dtype_pprinter(obj, p, cycle): if cycle: return p.text('dtype(...)') if hasattr(obj, 'fields'): if obj.fields is None: p.text(repr(obj)) else: p.begin_group(7, 'dtype([') for i, field in enumerate(obj.descr): if i > 0: p.text(',') p.breakable() p.pretty(field) p.end_group(7, '])') """ # The format type of data returned. format_type = Unicode('text/plain') # This subclass ignores this attribute as it always need to return # something. enabled = Bool(True, config=False) # Look for a _repr_pretty_ methods to use for pretty printing. print_method = ObjectName('_repr_pretty_') # Whether to pretty-print or not. pprint = Bool(True, config=True) # Whether to be verbose or not. verbose = Bool(False, config=True) # The maximum width. max_width = Integer(79, config=True) # The newline character. newline = Unicode('\n', config=True) # format-string for pprinting floats float_format = Unicode('%r') # setter for float precision, either int or direct format-string float_precision = CUnicode('', config=True) def _float_precision_changed(self, name, old, new): """float_precision changed, set float_format accordingly. float_precision can be set by int or str. This will set float_format, after interpreting input. If numpy has been imported, numpy print precision will also be set. integer `n` sets format to '%.nf', otherwise, format set directly. An empty string returns to defaults (repr for float, 8 for numpy). This parameter can be set via the '%precision' magic. """ if '%' in new: # got explicit format string fmt = new try: fmt % 3.14159 except Exception: raise ValueError( "Precision must be int or format string, not %r" % new) elif new: # otherwise, should be an int try: i = int(new) assert i >= 0 except ValueError: raise ValueError( "Precision must be int or format string, not %r" % new) except AssertionError: raise ValueError("int precision must be non-negative, not %r" % i) fmt = '%%.%if' % i if 'numpy' in sys.modules: # set numpy precision if it has been imported import numpy numpy.set_printoptions(precision=i) else: # default back to repr fmt = '%r' if 'numpy' in sys.modules: import numpy # numpy default is 8 numpy.set_printoptions(precision=8) self.float_format = fmt # Use the default pretty printers from IPython.lib.pretty. def _singleton_printers_default(self): return pretty._singleton_pprinters.copy() def _type_printers_default(self): d = pretty._type_pprinters.copy() d[float] = lambda obj, p, cycle: p.text(self.float_format % obj) return d def _deferred_printers_default(self): return pretty._deferred_type_pprinters.copy() #### FormatterABC interface #### def __call__(self, obj): """Compute the pretty representation of the object.""" if not self.pprint: try: return repr(obj) except TypeError: return '' else: # This uses use StringIO, as cStringIO doesn't handle unicode. stream = StringIO() # self.newline.encode() is a quick fix for issue gh-597. We need to # ensure that stream does not get a mix of unicode and bytestrings, # or it will cause trouble. printer = pretty.RepresentationPrinter( stream, self.verbose, self.max_width, unicode_to_str(self.newline), singleton_pprinters=self.singleton_printers, type_pprinters=self.type_printers, deferred_pprinters=self.deferred_printers) printer.pretty(obj) printer.flush() return stream.getvalue()
class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin): """ A Qt frontend for a generic Python kernel. """ # The text to show when the kernel is (re)started. banner = Unicode(config=True) # An option and corresponding signal for overriding the default kernel # interrupt behavior. custom_interrupt = Bool(False) custom_interrupt_requested = QtCore.Signal() # An option and corresponding signals for overriding the default kernel # restart behavior. custom_restart = Bool(False) custom_restart_kernel_died = QtCore.Signal(float) custom_restart_requested = QtCore.Signal() # Whether to automatically show calltips on open-parentheses. enable_calltips = Bool( True, config=True, help="Whether to draw information calltips on open-parentheses.") clear_on_kernel_restart = Bool( True, config=True, help="Whether to clear the console when the kernel is restarted") confirm_restart = Bool( True, config=True, help="Whether to ask for user confirmation when restarting kernel") # Emitted when a user visible 'execute_request' has been submitted to the # kernel from the FrontendWidget. Contains the code to be executed. executing = QtCore.Signal(object) # Emitted when a user-visible 'execute_reply' has been received from the # kernel and processed by the FrontendWidget. Contains the response message. executed = QtCore.Signal(object) # Emitted when an exit request has been received from the kernel. exit_requested = QtCore.Signal(object) # Protected class variables. _prompt_transformer = IPythonInputSplitter( physical_line_transforms=[classic_prompt()], logical_line_transforms=[], python_line_transforms=[], ) _CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos']) _CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos']) _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind']) _input_splitter_class = InputSplitter _local_kernel = False _highlighter = Instance(FrontendHighlighter) #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(FrontendWidget, self).__init__(*args, **kw) # FIXME: remove this when PySide min version is updated past 1.0.7 # forcefully disable calltips if PySide is < 1.0.7, because they crash if qt.QT_API == qt.QT_API_PYSIDE: import PySide if PySide.__version_info__ < (1, 0, 7): self.log.warn( "PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__) self.enable_calltips = False # FrontendWidget protected variables. self._bracket_matcher = BracketMatcher(self._control) self._call_tip_widget = CallTipWidget(self._control) self._completion_lexer = CompletionLexer(PythonLexer()) self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None) self._hidden = False self._highlighter = FrontendHighlighter(self) self._input_splitter = self._input_splitter_class() self._kernel_manager = None self._kernel_client = None self._request_info = {} self._request_info['execute'] = {} self._callback_dict = {} # Configure the ConsoleWidget. self.tab_width = 4 self._set_continuation_prompt('... ') # Configure the CallTipWidget. self._call_tip_widget.setFont(self.font) self.font_changed.connect(self._call_tip_widget.setFont) # Configure actions. action = self._copy_raw_action key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C action.setEnabled(False) action.setShortcut(QtGui.QKeySequence(key)) action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut) action.triggered.connect(self.copy_raw) self.copy_available.connect(action.setEnabled) self.addAction(action) # Connect signal handlers. document = self._control.document() document.contentsChange.connect(self._document_contents_change) # Set flag for whether we are connected via localhost. self._local_kernel = kw.get('local_kernel', FrontendWidget._local_kernel) #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- def copy(self): """ Copy the currently selected text to the clipboard, removing prompts. """ if self._page_control is not None and self._page_control.hasFocus(): self._page_control.copy() elif self._control.hasFocus(): text = self._control.textCursor().selection().toPlainText() if text: text = self._prompt_transformer.transform_cell(text) QtGui.QApplication.clipboard().setText(text) else: self.log.debug("frontend widget : unknown copy target") #--------------------------------------------------------------------------- # 'ConsoleWidget' abstract interface #--------------------------------------------------------------------------- def _is_complete(self, source, interactive): """ Returns whether 'source' can be completely processed and a new prompt created. When triggered by an Enter/Return key press, 'interactive' is True; otherwise, it is False. """ self._input_splitter.reset() complete = self._input_splitter.push(source) if interactive: complete = not self._input_splitter.push_accepts_more() return complete def _execute(self, source, hidden): """ Execute 'source'. If 'hidden', do not show any output. See parent class :meth:`execute` docstring for full details. """ msg_id = self.kernel_client.execute(source, hidden) self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'user') self._hidden = hidden if not hidden: self.executing.emit(source) def _prompt_started_hook(self): """ Called immediately after a new prompt is displayed. """ if not self._reading: self._highlighter.highlighting_on = True def _prompt_finished_hook(self): """ Called immediately after a prompt is finished, i.e. when some input will be processed and a new prompt displayed. """ # Flush all state from the input splitter so the next round of # reading input starts with a clean buffer. self._input_splitter.reset() if not self._reading: self._highlighter.highlighting_on = False def _tab_pressed(self): """ Called when the tab key is pressed. Returns whether to continue processing the event. """ # Perform tab completion if: # 1) The cursor is in the input buffer. # 2) There is a non-whitespace character before the cursor. text = self._get_input_buffer_cursor_line() if text is None: return False complete = bool(text[:self._get_input_buffer_cursor_column()].strip()) if complete: self._complete() return not complete #--------------------------------------------------------------------------- # 'ConsoleWidget' protected interface #--------------------------------------------------------------------------- def _context_menu_make(self, pos): """ Reimplemented to add an action for raw copy. """ menu = super(FrontendWidget, self)._context_menu_make(pos) for before_action in menu.actions(): if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \ QtGui.QKeySequence.ExactMatch: menu.insertAction(before_action, self._copy_raw_action) break return menu def request_interrupt_kernel(self): if self._executing: self.interrupt_kernel() def request_restart_kernel(self): message = 'Are you sure you want to restart the kernel?' self.restart_kernel(message, now=False) def _event_filter_console_keypress(self, event): """ Reimplemented for execution interruption and smart backspace. """ key = event.key() if self._control_key_down(event.modifiers(), include_command=False): if key == QtCore.Qt.Key_C and self._executing: self.request_interrupt_kernel() return True elif key == QtCore.Qt.Key_Period: self.request_restart_kernel() return True elif not event.modifiers() & QtCore.Qt.AltModifier: # Smart backspace: remove four characters in one backspace if: # 1) everything left of the cursor is whitespace # 2) the four characters immediately left of the cursor are spaces if key == QtCore.Qt.Key_Backspace: col = self._get_input_buffer_cursor_column() cursor = self._control.textCursor() if col > 3 and not cursor.hasSelection(): text = self._get_input_buffer_cursor_line()[:col] if text.endswith(' ') and not text.strip(): cursor.movePosition(QtGui.QTextCursor.Left, QtGui.QTextCursor.KeepAnchor, 4) cursor.removeSelectedText() return True return super(FrontendWidget, self)._event_filter_console_keypress(event) def _insert_continuation_prompt(self, cursor): """ Reimplemented for auto-indentation. """ super(FrontendWidget, self)._insert_continuation_prompt(cursor) cursor.insertText(' ' * self._input_splitter.indent_spaces) #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_complete_reply(self, rep): """ Handle replies for tab completion. """ self.log.debug("complete: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('complete') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): text = '.'.join(self._get_context()) cursor.movePosition(QtGui.QTextCursor.Left, n=len(text)) self._complete_with_items(cursor, rep['content']['matches']) def _silent_exec_callback(self, expr, callback): """Silently execute `expr` in the kernel and call `callback` with reply the `expr` is evaluated silently in the kernel (without) output in the frontend. Call `callback` with the `repr <http://docs.python.org/library/functions.html#repr> `_ as first argument Parameters ---------- expr : string valid string to be executed by the kernel. callback : function function accepting one argument, as a string. The string will be the `repr` of the result of evaluating `expr` The `callback` is called with the `repr()` of the result of `expr` as first argument. To get the object, do `eval()` on the passed value. See Also -------- _handle_exec_callback : private method, deal with calling callback with reply """ # generate uuid, which would be used as an indication of whether or # not the unique request originated from here (can use msg id ?) local_uuid = str(uuid.uuid1()) msg_id = self.kernel_client.execute( '', silent=True, user_expressions={local_uuid: expr}) self._callback_dict[local_uuid] = callback self._request_info['execute'][msg_id] = self._ExecutionRequest( msg_id, 'silent_exec_callback') def _handle_exec_callback(self, msg): """Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback`` Parameters ---------- msg : raw message send by the kernel containing an `user_expressions` and having a 'silent_exec_callback' kind. Notes ----- This function will look for a `callback` associated with the corresponding message id. Association has been made by `_silent_exec_callback`. `callback` is then called with the `repr()` of the value of corresponding `user_expressions` as argument. `callback` is then removed from the known list so that any message coming again with the same id won't trigger it. """ user_exp = msg['content'].get('user_expressions') if not user_exp: return for expression in user_exp: if expression in self._callback_dict: self._callback_dict.pop(expression)(user_exp[expression]) def _handle_execute_reply(self, msg): """ Handles replies for code execution. """ self.log.debug("execute: %s", msg.get('content', '')) msg_id = msg['parent_header']['msg_id'] info = self._request_info['execute'].get(msg_id) # unset reading flag, because if execute finished, raw_input can't # still be pending. self._reading = False if info and info.kind == 'user' and not self._hidden: # Make sure that all output from the SUB channel has been processed # before writing a new prompt. self.kernel_client.iopub_channel.flush() # Reset the ANSI style information to prevent bad text in stdout # from messing up our colors. We're not a true terminal so we're # allowed to do this. if self.ansi_codes: self._ansi_processor.reset_sgr() content = msg['content'] status = content['status'] if status == 'ok': self._process_execute_ok(msg) elif status == 'error': self._process_execute_error(msg) elif status == 'aborted': self._process_execute_abort(msg) self._show_interpreter_prompt_for_reply(msg) self.executed.emit(msg) self._request_info['execute'].pop(msg_id) elif info and info.kind == 'silent_exec_callback' and not self._hidden: self._handle_exec_callback(msg) self._request_info['execute'].pop(msg_id) else: super(FrontendWidget, self)._handle_execute_reply(msg) def _handle_input_request(self, msg): """ Handle requests for raw_input. """ self.log.debug("input: %s", msg.get('content', '')) if self._hidden: raise RuntimeError( 'Request for raw input during hidden execution.') # Make sure that all output from the SUB channel has been processed # before entering readline mode. self.kernel_client.iopub_channel.flush() def callback(line): self.kernel_client.stdin_channel.input(line) if self._reading: self.log.debug( "Got second input request, assuming first was interrupted.") self._reading = False self._readline(msg['content']['prompt'], callback=callback) def _kernel_restarted_message(self, died=True): msg = "Kernel died, restarting" if died else "Kernel restarting" self._append_html("<br>%s<hr><br>" % msg, before_prompt=False) def _handle_kernel_died(self, since_last_heartbeat): """Handle the kernel's death (if we do not own the kernel). """ self.log.warn("kernel died: %s", since_last_heartbeat) if self.custom_restart: self.custom_restart_kernel_died.emit(since_last_heartbeat) else: self._kernel_restarted_message(died=True) self.reset() def _handle_kernel_restarted(self, died=True): """Notice that the autorestarter restarted the kernel. There's nothing to do but show a message. """ self.log.warn("kernel restarted") self._kernel_restarted_message(died=died) self.reset() def _handle_object_info_reply(self, rep): """ Handle replies for call tips. """ self.log.debug("oinfo: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('call_tip') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): # Get the information for a call tip. For now we format the call # line as string, later we can pass False to format_call and # syntax-highlight it ourselves for nicer formatting in the # calltip. content = rep['content'] # if this is from pykernel, 'docstring' will be the only key if content.get('ismagic', False): # Don't generate a call-tip for magics. Ideally, we should # generate a tooltip, but not on ( like we do for actual # callables. call_info, doc = None, None else: call_info, doc = call_tip(content, format_call=True) if call_info or doc: self._call_tip_widget.show_call_info(call_info, doc) def _handle_pyout(self, msg): """ Handle display hook output. """ self.log.debug("pyout: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): text = msg['content']['data'] self._append_plain_text(text + '\n', before_prompt=True) def _handle_stream(self, msg): """ Handle stdout, stderr, and stdin. """ self.log.debug("stream: %s", msg.get('content', '')) if not self._hidden and self._is_from_this_session(msg): # Most consoles treat tabs as being 8 space characters. Convert tabs # to spaces so that output looks as expected regardless of this # widget's tab width. text = msg['content']['data'].expandtabs(8) self._append_plain_text(text, before_prompt=True) self._control.moveCursor(QtGui.QTextCursor.End) def _handle_shutdown_reply(self, msg): """ Handle shutdown signal, only if from other console. """ self.log.warn("shutdown: %s", msg.get('content', '')) restart = msg.get('content', {}).get('restart', False) if not self._hidden and not self._is_from_this_session(msg): # got shutdown reply, request came from session other than ours if restart: # someone restarted the kernel, handle it self._handle_kernel_restarted(died=False) else: # kernel was shutdown permanently # this triggers exit_requested if the kernel was local, # and a dialog if the kernel was remote, # so we don't suddenly clear the qtconsole without asking. if self._local_kernel: self.exit_requested.emit(self) else: title = self.window().windowTitle() reply = QtGui.QMessageBox.question( self, title, "Kernel has been shutdown permanently. " "Close the Console?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.Yes: self.exit_requested.emit(self) def _handle_status(self, msg): """Handle status message""" # This is where a busy/idle indicator would be triggered, # when we make one. state = msg['content'].get('execution_state', '') if state == 'starting': # kernel started while we were running if self._executing: self._handle_kernel_restarted(died=True) elif state == 'idle': pass elif state == 'busy': pass def _started_channels(self): """ Called when the KernelManager channels have started listening or when the frontend is assigned an already listening KernelManager. """ self.reset(clear=True) #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def copy_raw(self): """ Copy the currently selected text to the clipboard without attempting to remove prompts or otherwise alter the text. """ self._control.copy() def execute_file(self, path, hidden=False): """ Attempts to execute file with 'path'. If 'hidden', no output is shown. """ self.execute('execfile(%r)' % path, hidden=hidden) def interrupt_kernel(self): """ Attempts to interrupt the running kernel. Also unsets _reading flag, to avoid runtime errors if raw_input is called again. """ if self.custom_interrupt: self._reading = False self.custom_interrupt_requested.emit() elif self.kernel_manager: self._reading = False self.kernel_manager.interrupt_kernel() else: self._append_plain_text( 'Cannot interrupt a kernel I did not start.\n') def reset(self, clear=False): """ Resets the widget to its initial state if ``clear`` parameter is True, otherwise prints a visual indication of the fact that the kernel restarted, but does not clear the traces from previous usage of the kernel before it was restarted. With ``clear=True``, it is similar to ``%clear``, but also re-writes the banner and aborts execution if necessary. """ if self._executing: self._executing = False self._request_info['execute'] = {} self._reading = False self._highlighter.highlighting_on = False if clear: self._control.clear() self._append_plain_text(self.banner) # update output marker for stdout/stderr, so that startup # messages appear after banner: self._append_before_prompt_pos = self._get_cursor().position() self._show_interpreter_prompt() def restart_kernel(self, message, now=False): """ Attempts to restart the running kernel. """ # FIXME: now should be configurable via a checkbox in the dialog. Right # now at least the heartbeat path sets it to True and the manual restart # to False. But those should just be the pre-selected states of a # checkbox that the user could override if so desired. But I don't know # enough Qt to go implementing the checkbox now. if self.custom_restart: self.custom_restart_requested.emit() return if self.kernel_manager: # Pause the heart beat channel to prevent further warnings. self.kernel_client.hb_channel.pause() # Prompt the user to restart the kernel. Un-pause the heartbeat if # they decline. (If they accept, the heartbeat will be un-paused # automatically when the kernel is restarted.) if self.confirm_restart: buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No result = QtGui.QMessageBox.question(self, 'Restart kernel?', message, buttons) do_restart = result == QtGui.QMessageBox.Yes else: # confirm_restart is False, so we don't need to ask user # anything, just do the restart do_restart = True if do_restart: try: self.kernel_manager.restart_kernel(now=now) except RuntimeError as e: self._append_plain_text('Error restarting kernel: %s\n' % e, before_prompt=True) else: self._append_html( "<br>Restarting kernel...\n<hr><br>", before_prompt=True, ) else: self.kernel_client.hb_channel.unpause() else: self._append_plain_text( 'Cannot restart a Kernel I did not start\n', before_prompt=True) #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _call_tip(self): """ Shows a call tip, if appropriate, at the current cursor location. """ # Decide if it makes sense to show a call tip if not self.enable_calltips: return False cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.Left) if cursor.document().characterAt(cursor.position()) != '(': return False context = self._get_context(cursor) if not context: return False # Send the metadata request to the kernel name = '.'.join(context) msg_id = self.kernel_client.object_info(name) pos = self._get_cursor().position() self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos) return True def _complete(self): """ Performs completion at the current cursor location. """ context = self._get_context() if context: # Send the completion request to the kernel msg_id = self.kernel_client.complete( '.'.join(context), # text self._get_input_buffer_cursor_line(), # line self._get_input_buffer_cursor_column(), # cursor_pos self.input_buffer) # block pos = self._get_cursor().position() info = self._CompletionRequest(msg_id, pos) self._request_info['complete'] = info def _get_context(self, cursor=None): """ Gets the context for the specified cursor (or the current cursor if none is specified). """ if cursor is None: cursor = self._get_cursor() cursor.movePosition(QtGui.QTextCursor.StartOfBlock, QtGui.QTextCursor.KeepAnchor) text = cursor.selection().toPlainText() return self._completion_lexer.get_context(text) def _process_execute_abort(self, msg): """ Process a reply for an aborted execution request. """ self._append_plain_text("ERROR: execution aborted\n") def _process_execute_error(self, msg): """ Process a reply for an execution request that resulted in an error. """ content = msg['content'] # If a SystemExit is passed along, this means exit() was called - also # all the ipython %exit magic syntax of '-k' to be used to keep # the kernel running if content['ename'] == 'SystemExit': keepkernel = content['evalue'] == '-k' or content[ 'evalue'] == 'True' self._keep_kernel_on_exit = keepkernel self.exit_requested.emit(self) else: traceback = ''.join(content['traceback']) self._append_plain_text(traceback) def _process_execute_ok(self, msg): """ Process a reply for a successful execution request. """ payload = msg['content']['payload'] for item in payload: if not self._process_execute_payload(item): warning = 'Warning: received unknown payload of type %s' print(warning % repr(item['source'])) def _process_execute_payload(self, item): """ Process a single payload item from the list of payload items in an execution reply. Returns whether the payload was handled. """ # The basic FrontendWidget doesn't handle payloads, as they are a # mechanism for going beyond the standard Python interpreter model. return False def _show_interpreter_prompt(self): """ Shows a prompt for the interpreter. """ self._show_prompt('>>> ') def _show_interpreter_prompt_for_reply(self, msg): """ Shows a prompt for the interpreter given an 'execute_reply' message. """ self._show_interpreter_prompt() #------ Signal handlers ---------------------------------------------------- def _document_contents_change(self, position, removed, added): """ Called whenever the document's content changes. Display a call tip if appropriate. """ # Calculate where the cursor should be *after* the change: position += added document = self._control.document() if position == self._get_cursor().position(): self._call_tip() #------ Trait default initializers ----------------------------------------- def _banner_default(self): """ Returns the standard Python banner. """ banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \ '"license" for more information.' return banner % (sys.version, sys.platform)
class DisplayFormatter(Configurable): # When set to true only the default plain text formatter will be used. plain_text_only = Bool(False, config=True) def _plain_text_only_changed(self, name, old, new): warnings.warn( """DisplayFormatter.plain_text_only is deprecated. Use DisplayFormatter.active_types = ['text/plain'] for the same effect. """, DeprecationWarning) if new: self.active_types = ['text/plain'] else: self.active_types = self.format_types active_types = List(Unicode, config=True, help="""List of currently active mime-types to display. You can use this to set a white-list for formats to display. Most users will not need to change this value. """) def _active_types_default(self): return self.format_types def _active_types_changed(self, name, old, new): for key, formatter in list(self.formatters.items()): if key in new: formatter.enabled = True else: formatter.enabled = False # A dict of formatter whose keys are format types (MIME types) and whose # values are subclasses of BaseFormatter. formatters = Dict() def _formatters_default(self): """Activate the default formatters.""" formatter_classes = [ PlainTextFormatter, HTMLFormatter, SVGFormatter, PNGFormatter, JPEGFormatter, LatexFormatter, JSONFormatter, JavascriptFormatter ] d = {} for cls in formatter_classes: f = cls(parent=self) d[f.format_type] = f return d def format(self, obj, include=None, exclude=None): """Return a format data dict for an object. By default all format types will be computed. The following MIME types are currently implemented: * text/plain * text/html * text/latex * application/json * application/javascript * image/png * image/jpeg * image/svg+xml Parameters ---------- obj : object The Python object whose format data will be computed. include : list or tuple, optional A list of format type strings (MIME types) to include in the format data dict. If this is set *only* the format types included in this list will be computed. exclude : list or tuple, optional A list of format type string (MIME types) to exclude in the format data dict. If this is set all format types will be computed, except for those included in this argument. Returns ------- (format_dict, metadata_dict) : tuple of two dicts format_dict is a dictionary of key/value pairs, one of each format that was generated for the object. The keys are the format types, which will usually be MIME type strings and the values and JSON'able data structure containing the raw data for the representation in that format. metadata_dict is a dictionary of metadata about each mime-type output. Its keys will be a strict subset of the keys in format_dict. """ format_dict = {} md_dict = {} for format_type, formatter in list(self.formatters.items()): if include and format_type not in include: continue if exclude and format_type in exclude: continue md = None try: data = formatter(obj) except: # FIXME: log the exception raise # formatters can return raw data or (data, metadata) if isinstance(data, tuple) and len(data) == 2: data, md = data if data is not None: format_dict[format_type] = data if md is not None: md_dict[format_type] = md return format_dict, md_dict @property def format_types(self): """Return the format types (MIME types) of the active formatters.""" return list(self.formatters.keys())
class IPythonWidget(FrontendWidget): """ A FrontendWidget for an IPython kernel. """ # If set, the 'custom_edit_requested(str, int)' signal will be emitted when # an editor is needed for a file. This overrides 'editor' and 'editor_line' # settings. custom_edit = Bool(False) custom_edit_requested = QtCore.Signal(object, object) editor = Unicode(default_editor, config=True, help=""" A command for invoking a system text editor. If the string contains a {filename} format specifier, it will be used. Otherwise, the filename will be appended to the end the command. """) editor_line = Unicode(config=True, help=""" The editor command to use when a specific line number is requested. The string should contain two format specifiers: {line} and {filename}. If this parameter is not specified, the line number option to the %edit magic will be ignored. """) style_sheet = Unicode(config=True, help=""" A CSS stylesheet. The stylesheet can contain classes for: 1. Qt: QPlainTextEdit, QFrame, QWidget, etc 2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter) 3. IPython: .error, .in-prompt, .out-prompt, etc """) syntax_style = Unicode(config=True, help=""" If not empty, use this Pygments style for syntax highlighting. Otherwise, the style sheet is queried for Pygments style information. """) # Prompts. in_prompt = Unicode(default_in_prompt, config=True) out_prompt = Unicode(default_out_prompt, config=True) input_sep = Unicode(default_input_sep, config=True) output_sep = Unicode(default_output_sep, config=True) output_sep2 = Unicode(default_output_sep2, config=True) # FrontendWidget protected class variables. _input_splitter_class = IPythonInputSplitter _prompt_transformer = IPythonInputSplitter( physical_line_transforms=[ipy_prompt()], logical_line_transforms=[], python_line_transforms=[], ) # IPythonWidget protected class variables. _PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number']) _payload_source_edit = 'edit' _payload_source_exit = 'ask_exit' _payload_source_next_input = 'set_next_input' _payload_source_page = 'page' _retrying_history_request = False _starting = False #--------------------------------------------------------------------------- # 'object' interface #--------------------------------------------------------------------------- def __init__(self, *args, **kw): super(IPythonWidget, self).__init__(*args, **kw) # IPythonWidget protected variables. self._payload_handlers = { self._payload_source_edit: self._handle_payload_edit, self._payload_source_exit: self._handle_payload_exit, self._payload_source_page: self._handle_payload_page, self._payload_source_next_input: self._handle_payload_next_input } self._previous_prompt_obj = None self._keep_kernel_on_exit = None # Initialize widget styling. if self.style_sheet: self._style_sheet_changed() self._syntax_style_changed() else: self.set_default_style() self._guiref_loaded = False #--------------------------------------------------------------------------- # 'BaseFrontendMixin' abstract interface #--------------------------------------------------------------------------- def _handle_complete_reply(self, rep): """ Reimplemented to support IPython's improved completion machinery. """ self.log.debug("complete: %s", rep.get('content', '')) cursor = self._get_cursor() info = self._request_info.get('complete') if info and info.id == rep['parent_header']['msg_id'] and \ info.pos == cursor.position(): content = rep['content'] matches = content['matches'] start = content['cursor_start'] end = content['cursor_end'] start = max(start, 0) end = max(end, start) # Move the control's cursor to the desired end point cursor_pos = self._get_input_buffer_cursor_pos() if end < cursor_pos: cursor.movePosition(QtGui.QTextCursor.Left, n=(cursor_pos - end)) elif end > cursor_pos: cursor.movePosition(QtGui.QTextCursor.Right, n=(end - cursor_pos)) # This line actually applies the move to control's cursor self._control.setTextCursor(cursor) offset = end - start # Move the local cursor object to the start of the match and # complete. cursor.movePosition(QtGui.QTextCursor.Left, n=offset) self._complete_with_items(cursor, matches) def _handle_execute_reply(self, msg): """ Reimplemented to support prompt requests. """ msg_id = msg['parent_header'].get('msg_id') info = self._request_info['execute'].get(msg_id) if info and info.kind == 'prompt': content = msg['content'] if content['status'] == 'aborted': self._show_interpreter_prompt() else: number = content['execution_count'] + 1 self._show_interpreter_prompt(number) self._request_info['execute'].pop(msg_id) else: super(IPythonWidget, self)._handle_execute_reply(msg) def _handle_history_reply(self, msg): """ Implemented to handle history tail replies, which are only supported by the IPython kernel. """ content = msg['content'] if 'history' not in content: self.log.error("History request failed: %r" % content) if content.get('status', '') == 'aborted' and \ not self._retrying_history_request: # a *different* action caused this request to be aborted, so # we should try again. self.log.error("Retrying aborted history request") # prevent multiple retries of aborted requests: self._retrying_history_request = True # wait out the kernel's queue flush, which is currently timed at 0.1s time.sleep(0.25) self.kernel_client.shell_channel.history( hist_access_type='tail', n=1000) else: self._retrying_history_request = False return # reset retry flag self._retrying_history_request = False history_items = content['history'] self.log.debug("Received history reply with %i entries", len(history_items)) items = [] last_cell = u"" for _, _, cell in history_items: cell = cell.rstrip() if cell != last_cell: items.append(cell) last_cell = cell self._set_history(items) def _insert_other_input(self, cursor, content): """Insert function for input from other frontends""" cursor.beginEditBlock() start = cursor.position() n = content.get('execution_count', 0) cursor.insertText('\n') self._insert_html(cursor, self._make_in_prompt(n)) cursor.insertText(content['code']) self._highlighter.rehighlightBlock(cursor.block()) cursor.endEditBlock() def _handle_execute_input(self, msg): """Handle an execute_input message""" self.log.debug("execute_input: %s", msg.get('content', '')) if self.include_output(msg): self._append_custom(self._insert_other_input, msg['content'], before_prompt=True) def _handle_execute_result(self, msg): """ Reimplemented for IPython-style "display hook". """ self.log.debug("execute_result: %s", msg.get('content', '')) if self.include_output(msg): self.flush_clearoutput() content = msg['content'] prompt_number = content.get('execution_count', 0) data = content['data'] if 'text/plain' in data: self._append_plain_text(self.output_sep, True) self._append_html(self._make_out_prompt(prompt_number), True) text = data['text/plain'] # If the repr is multiline, make sure we start on a new line, # so that its lines are aligned. if "\n" in text and not self.output_sep.endswith("\n"): self._append_plain_text('\n', True) self._append_plain_text(text + self.output_sep2, True) def _handle_display_data(self, msg): """ The base handler for the ``display_data`` message. """ self.log.debug("display: %s", msg.get('content', '')) # For now, we don't display data from other frontends, but we # eventually will as this allows all frontends to monitor the display # data. But we need to figure out how to handle this in the GUI. if self.include_output(msg): self.flush_clearoutput() data = msg['content']['data'] metadata = msg['content']['metadata'] # In the regular IPythonWidget, we simply print the plain text # representation. if 'text/plain' in data: text = data['text/plain'] self._append_plain_text(text, True) # This newline seems to be needed for text and html output. self._append_plain_text(u'\n', True) def _handle_kernel_info_reply(self, rep): """Handle kernel info replies.""" content = rep['content'] if not self._guiref_loaded: if content.get('language') == 'python': self._load_guiref_magic() self._guiref_loaded = True self.kernel_banner = content.get('banner', '') if self._starting: # finish handling started channels self._starting = False super(IPythonWidget, self)._started_channels() def _started_channels(self): """Reimplemented to make a history request and load %guiref.""" self._starting = True # The reply will trigger %guiref load provided language=='python' self.kernel_client.kernel_info() self.kernel_client.shell_channel.history(hist_access_type='tail', n=1000) def _load_guiref_magic(self): """Load %guiref magic.""" self.kernel_client.shell_channel.execute('\n'.join([ "try:", " _usage", "except:", " from IPython.core import usage as _usage", " get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')", " del _usage", ]), silent=True) #--------------------------------------------------------------------------- # 'ConsoleWidget' public interface #--------------------------------------------------------------------------- #--------------------------------------------------------------------------- # 'FrontendWidget' public interface #--------------------------------------------------------------------------- def execute_file(self, path, hidden=False): """ Reimplemented to use the 'run' magic. """ # Use forward slashes on Windows to avoid escaping each separator. if sys.platform == 'win32': path = os.path.normpath(path).replace('\\', '/') # Perhaps we should not be using %run directly, but while we # are, it is necessary to quote or escape filenames containing spaces # or quotes. # In earlier code here, to minimize escaping, we sometimes quoted the # filename with single quotes. But to do this, this code must be # platform-aware, because run uses shlex rather than python string # parsing, so that: # * In Win: single quotes can be used in the filename without quoting, # and we cannot use single quotes to quote the filename. # * In *nix: we can escape double quotes in a double quoted filename, # but can't escape single quotes in a single quoted filename. # So to keep this code non-platform-specific and simple, we now only # use double quotes to quote filenames, and escape when needed: if ' ' in path or "'" in path or '"' in path: path = '"%s"' % path.replace('"', '\\"') self.execute('%%run %s' % path, hidden=hidden) #--------------------------------------------------------------------------- # 'FrontendWidget' protected interface #--------------------------------------------------------------------------- def _process_execute_error(self, msg): """ Reimplemented for IPython-style traceback formatting. """ content = msg['content'] traceback = '\n'.join(content['traceback']) + '\n' if False: # FIXME: For now, tracebacks come as plain text, so we can't use # the html renderer yet. Once we refactor ultratb to produce # properly styled tracebacks, this branch should be the default traceback = traceback.replace(' ', ' ') traceback = traceback.replace('\n', '<br/>') ename = content['ename'] ename_styled = '<span class="error">%s</span>' % ename traceback = traceback.replace(ename, ename_styled) self._append_html(traceback) else: # This is the fallback for now, using plain text with ansi escapes self._append_plain_text(traceback) def _process_execute_payload(self, item): """ Reimplemented to dispatch payloads to handler methods. """ handler = self._payload_handlers.get(item['source']) if handler is None: # We have no handler for this type of payload, simply ignore it return False else: handler(item) return True def _show_interpreter_prompt(self, number=None): """ Reimplemented for IPython-style prompts. """ # If a number was not specified, make a prompt number request. if number is None: msg_id = self.kernel_client.shell_channel.execute('', silent=True) info = self._ExecutionRequest(msg_id, 'prompt') self._request_info['execute'][msg_id] = info return # Show a new prompt and save information about it so that it can be # updated later if the prompt number turns out to be wrong. self._prompt_sep = self.input_sep self._show_prompt(self._make_in_prompt(number), html=True) block = self._control.document().lastBlock() length = len(self._prompt) self._previous_prompt_obj = self._PromptBlock(block, length, number) # Update continuation prompt to reflect (possibly) new prompt length. self._set_continuation_prompt(self._make_continuation_prompt( self._prompt), html=True) def _show_interpreter_prompt_for_reply(self, msg): """ Reimplemented for IPython-style prompts. """ # Update the old prompt number if necessary. content = msg['content'] # abort replies do not have any keys: if content['status'] == 'aborted': if self._previous_prompt_obj: previous_prompt_number = self._previous_prompt_obj.number else: previous_prompt_number = 0 else: previous_prompt_number = content['execution_count'] if self._previous_prompt_obj and \ self._previous_prompt_obj.number != previous_prompt_number: block = self._previous_prompt_obj.block # Make sure the prompt block has not been erased. if block.isValid() and block.text(): # Remove the old prompt and insert a new prompt. cursor = QtGui.QTextCursor(block) cursor.movePosition(QtGui.QTextCursor.Right, QtGui.QTextCursor.KeepAnchor, self._previous_prompt_obj.length) prompt = self._make_in_prompt(previous_prompt_number) self._prompt = self._insert_html_fetching_plain_text( cursor, prompt) # When the HTML is inserted, Qt blows away the syntax # highlighting for the line, so we need to rehighlight it. self._highlighter.rehighlightBlock(cursor.block()) self._previous_prompt_obj = None # Show a new prompt with the kernel's estimated prompt number. self._show_interpreter_prompt(previous_prompt_number + 1) #--------------------------------------------------------------------------- # 'IPythonWidget' interface #--------------------------------------------------------------------------- def set_default_style(self, colors='lightbg'): """ Sets the widget style to the class defaults. Parameters ---------- colors : str, optional (default lightbg) Whether to use the default IPython light background or dark background or B&W style. """ colors = colors.lower() if colors == 'lightbg': self.style_sheet = styles.default_light_style_sheet self.syntax_style = styles.default_light_syntax_style elif colors == 'linux': self.style_sheet = styles.default_dark_style_sheet self.syntax_style = styles.default_dark_syntax_style elif colors == 'nocolor': self.style_sheet = styles.default_bw_style_sheet self.syntax_style = styles.default_bw_syntax_style else: raise KeyError("No such color scheme: %s" % colors) #--------------------------------------------------------------------------- # 'IPythonWidget' protected interface #--------------------------------------------------------------------------- def _edit(self, filename, line=None): """ Opens a Python script for editing. Parameters ---------- filename : str A path to a local system file. line : int, optional A line of interest in the file. """ if self.custom_edit: self.custom_edit_requested.emit(filename, line) elif not self.editor: self._append_plain_text( 'No default editor available.\n' 'Specify a GUI text editor in the `IPythonWidget.editor` ' 'configurable to enable the %edit magic') else: try: filename = '"%s"' % filename if line and self.editor_line: command = self.editor_line.format(filename=filename, line=line) else: try: command = self.editor.format() except KeyError: command = self.editor.format(filename=filename) else: command += ' ' + filename except KeyError: self._append_plain_text('Invalid editor command.\n') else: try: Popen(command, shell=True) except OSError: msg = 'Opening editor with command "%s" failed.\n' self._append_plain_text(msg % command) def _make_in_prompt(self, number): """ Given a prompt number, returns an HTML In prompt. """ try: body = self.in_prompt % number except TypeError: # allow in_prompt to leave out number, e.g. '>>> ' from xml.sax.saxutils import escape body = escape(self.in_prompt) return '<span class="in-prompt">%s</span>' % body def _make_continuation_prompt(self, prompt): """ Given a plain text version of an In prompt, returns an HTML continuation prompt. """ end_chars = '...: ' space_count = len(prompt.lstrip('\n')) - len(end_chars) body = ' ' * space_count + end_chars return '<span class="in-prompt">%s</span>' % body def _make_out_prompt(self, number): """ Given a prompt number, returns an HTML Out prompt. """ try: body = self.out_prompt % number except TypeError: # allow out_prompt to leave out number, e.g. '<<< ' from xml.sax.saxutils import escape body = escape(self.out_prompt) return '<span class="out-prompt">%s</span>' % body #------ Payload handlers -------------------------------------------------- # Payload handlers with a generic interface: each takes the opaque payload # dict, unpacks it and calls the underlying functions with the necessary # arguments. def _handle_payload_edit(self, item): self._edit(item['filename'], item['line_number']) def _handle_payload_exit(self, item): self._keep_kernel_on_exit = item['keepkernel'] self.exit_requested.emit(self) def _handle_payload_next_input(self, item): self.input_buffer = item['text'] def _handle_payload_page(self, item): # Since the plain text widget supports only a very small subset of HTML # and we have no control over the HTML source, we only page HTML # payloads in the rich text widget. data = item['data'] if 'text/html' in data and self.kind == 'rich': self._page(data['text/html'], html=True) else: self._page(data['text/plain'], html=False) #------ Trait change handlers -------------------------------------------- def _style_sheet_changed(self): """ Set the style sheets of the underlying widgets. """ self.setStyleSheet(self.style_sheet) if self._control is not None: self._control.document().setDefaultStyleSheet(self.style_sheet) bg_color = self._control.palette().window().color() self._ansi_processor.set_background_color(bg_color) if self._page_control is not None: self._page_control.document().setDefaultStyleSheet( self.style_sheet) def _syntax_style_changed(self): """ Set the style for the syntax highlighter. """ if self._highlighter is None: # ignore premature calls return if self.syntax_style: self._highlighter.set_style(self.syntax_style) else: self._highlighter.set_style_sheet(self.style_sheet) #------ Trait default initializers ----------------------------------------- def _banner_default(self): return "IPython QtConsole {version}\n".format(version=version)
class ProfileCreate(BaseIPythonApplication): name = 'ipython-profile' description = create_help examples = _create_examples auto_create = Bool(True, config=False) def _log_format_default(self): return "[%(name)s] %(message)s" def _copy_config_files_default(self): return True parallel = Bool(False, config=True, help="whether to include parallel computing config files") def _parallel_changed(self, name, old, new): parallel_files = [ 'ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py' ] if new: for cf in parallel_files: self.config_files.append(cf) else: for cf in parallel_files: if cf in self.config_files: self.config_files.remove(cf) def parse_command_line(self, argv): super(ProfileCreate, self).parse_command_line(argv) # accept positional arg as profile name if self.extra_args: self.profile = self.extra_args[0] flags = Dict(create_flags) classes = [ProfileDir] def _import_app(self, app_path): """import an app class""" app = None name = app_path.rsplit('.', 1)[-1] try: app = import_item(app_path) except ImportError as e: self.log.info("Couldn't import %s, config file will be excluded", name) except Exception: self.log.warn('Unexpected error importing %s', name, exc_info=True) return app def init_config_files(self): super(ProfileCreate, self).init_config_files() # use local imports, since these classes may import from here from IPython.terminal.ipapp import TerminalIPythonApp apps = [TerminalIPythonApp] for app_path in ( 'IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', 'IPython.html.notebookapp.NotebookApp', 'IPython.nbconvert.nbconvertapp.NbConvertApp', ): app = self._import_app(app_path) if app is not None: apps.append(app) if self.parallel: from IPython.parallel.apps.ipcontrollerapp import IPControllerApp from IPython.parallel.apps.ipengineapp import IPEngineApp from IPython.parallel.apps.ipclusterapp import IPClusterStart from IPython.parallel.apps.iploggerapp import IPLoggerApp apps.extend([ IPControllerApp, IPEngineApp, IPClusterStart, IPLoggerApp, ]) for App in apps: app = App() app.config.update(self.config) app.log = self.log app.overwrite = self.overwrite app.copy_config_files = True app.ipython_dir = self.ipython_dir app.profile_dir = self.profile_dir app.init_config_files() def stage_default_config_file(self): pass
class ZMQTerminalInteractiveShell(TerminalInteractiveShell): """A subclass of TerminalInteractiveShell that uses the 0MQ kernel""" _executing = False _execution_state = Unicode('') _pending_clearoutput = False kernel_banner = Unicode('') kernel_timeout = Float(60, config=True, help="""Timeout for giving up on a kernel (in seconds). On first connect and restart, the console tests whether the kernel is running and responsive by sending kernel_info_requests. This sets the timeout in seconds for how long the kernel can take before being presumed dead. """ ) image_handler = Enum(('PIL', 'stream', 'tempfile', 'callable'), config=True, allow_none=True, help= """ Handler for image type output. This is useful, for example, when connecting to the kernel in which pylab inline backend is activated. There are four handlers defined. 'PIL': Use Python Imaging Library to popup image; 'stream': Use an external program to show the image. Image will be fed into the STDIN of the program. You will need to configure `stream_image_handler`; 'tempfile': Use an external program to show the image. Image will be saved in a temporally file and the program is called with the temporally file. You will need to configure `tempfile_image_handler`; 'callable': You can set any Python callable which is called with the image data. You will need to configure `callable_image_handler`. """ ) stream_image_handler = List(config=True, help= """ Command to invoke an image viewer program when you are using 'stream' image handler. This option is a list of string where the first element is the command itself and reminders are the options for the command. Raw image data is given as STDIN to the program. """ ) tempfile_image_handler = List(config=True, help= """ Command to invoke an image viewer program when you are using 'tempfile' image handler. This option is a list of string where the first element is the command itself and reminders are the options for the command. You can use {file} and {format} in the string to represent the location of the generated image file and image format. """ ) callable_image_handler = Any(config=True, help= """ Callable object called via 'callable' image handler with one argument, `data`, which is `msg["content"]["data"]` where `msg` is the message from iopub channel. For exmaple, you can find base64 encoded PNG data as `data['image/png']`. """ ) mime_preference = List( default_value=['image/png', 'image/jpeg', 'image/svg+xml'], config=True, help= """ Preferred object representation MIME type in order. First matched MIME type will be used. """ ) manager = Instance('IPython.kernel.KernelManager') client = Instance('IPython.kernel.KernelClient') def _client_changed(self, name, old, new): self.session_id = new.session.session session_id = Unicode() def init_completer(self): """Initialize the completion machinery. This creates completion machinery that can be used by client code, either interactively in-process (typically triggered by the readline library), programmatically (such as in test suites) or out-of-process (typically over the network by remote frontends). """ from IPython.core.completerlib import (module_completer, magic_run_completer, cd_completer) self.Completer = ZMQCompleter(self, self.client, config=self.config) self.set_hook('complete_command', module_completer, str_key = 'import') self.set_hook('complete_command', module_completer, str_key = 'from') self.set_hook('complete_command', magic_run_completer, str_key = '%run') self.set_hook('complete_command', cd_completer, str_key = '%cd') # Only configure readline if we truly are using readline. IPython can # do tab-completion over the network, in GUIs, etc, where readline # itself may be absent if self.has_readline: self.set_readline_completer() def run_cell(self, cell, store_history=True): """Run a complete IPython cell. Parameters ---------- cell : str The code (including IPython code such as %magic functions) to run. store_history : bool If True, the raw and translated cell will be stored in IPython's history. For user code calling back into IPython's machinery, this should be set to False. """ if (not cell) or cell.isspace(): # pressing enter flushes any pending display self.handle_iopub() return # flush stale replies, which could have been ignored, due to missed heartbeats while self.client.shell_channel.msg_ready(): self.client.shell_channel.get_msg() # execute takes 'hidden', which is the inverse of store_hist msg_id = self.client.execute(cell, not store_history) # first thing is wait for any side effects (output, stdin, etc.) self._executing = True self._execution_state = "busy" while self._execution_state != 'idle' and self.client.is_alive(): try: self.handle_input_request(msg_id, timeout=0.05) except Empty: # display intermediate print statements, etc. self.handle_iopub(msg_id) # after all of that is done, wait for the execute reply while self.client.is_alive(): try: self.handle_execute_reply(msg_id, timeout=0.05) except Empty: pass else: break self._executing = False #----------------- # message handlers #----------------- def handle_execute_reply(self, msg_id, timeout=None): msg = self.client.shell_channel.get_msg(block=False, timeout=timeout) if msg["parent_header"].get("msg_id", None) == msg_id: self.handle_iopub(msg_id) content = msg["content"] status = content['status'] if status == 'aborted': self.write('Aborted\n') return elif status == 'ok': # handle payloads for item in content["payload"]: source = item['source'] if source == 'page': page.page(item['data']['text/plain']) elif source == 'set_next_input': self.set_next_input(item['text']) elif source == 'ask_exit': self.ask_exit() elif status == 'error': for frame in content["traceback"]: print(frame, file=io.stderr) self.execution_count = int(content["execution_count"] + 1) include_other_output = Bool(False, config=True, help="""Whether to include output from clients other than this one sharing the same kernel. Outputs are not displayed until enter is pressed. """ ) other_output_prefix = Unicode("[remote] ", config=True, help="""Prefix to add to outputs coming from clients other than this one. Only relevant if include_other_output is True. """ ) def from_here(self, msg): """Return whether a message is from this session""" return msg['parent_header'].get("session", self.session_id) == self.session_id def include_output(self, msg): """Return whether we should include a given output message""" from_here = self.from_here(msg) if msg['msg_type'] == 'execute_input': # only echo inputs not from here return self.include_other_output and not from_here if self.include_other_output: return True else: return from_here def handle_iopub(self, msg_id=''): """Process messages on the IOPub channel This method consumes and processes messages on the IOPub channel, such as stdout, stderr, execute_result and status. It only displays output that is caused by this session. """ while self.client.iopub_channel.msg_ready(): sub_msg = self.client.iopub_channel.get_msg() msg_type = sub_msg['header']['msg_type'] parent = sub_msg["parent_header"] if self.include_output(sub_msg): if msg_type == 'status': self._execution_state = sub_msg["content"]["execution_state"] elif msg_type == 'stream': if sub_msg["content"]["name"] == "stdout": if self._pending_clearoutput: print("\r", file=io.stdout, end="") self._pending_clearoutput = False print(sub_msg["content"]["text"], file=io.stdout, end="") io.stdout.flush() elif sub_msg["content"]["name"] == "stderr": if self._pending_clearoutput: print("\r", file=io.stderr, end="") self._pending_clearoutput = False print(sub_msg["content"]["text"], file=io.stderr, end="") io.stderr.flush() elif msg_type == 'execute_result': if self._pending_clearoutput: print("\r", file=io.stdout, end="") self._pending_clearoutput = False self.execution_count = int(sub_msg["content"]["execution_count"]) if not self.from_here(sub_msg): sys.stdout.write(self.other_output_prefix) format_dict = sub_msg["content"]["data"] self.handle_rich_data(format_dict) # taken from DisplayHook.__call__: hook = self.displayhook hook.start_displayhook() hook.write_output_prompt() hook.write_format_data(format_dict) hook.log_output(format_dict) hook.finish_displayhook() elif msg_type == 'display_data': data = sub_msg["content"]["data"] handled = self.handle_rich_data(data) if not handled: if not self.from_here(sub_msg): sys.stdout.write(self.other_output_prefix) # if it was an image, we handled it by now if 'text/plain' in data: print(data['text/plain']) elif msg_type == 'execute_input': content = sub_msg['content'] self.execution_count = content['execution_count'] if not self.from_here(sub_msg): sys.stdout.write(self.other_output_prefix) sys.stdout.write(self.prompt_manager.render('in')) sys.stdout.write(content['code']) elif msg_type == 'clear_output': if sub_msg["content"]["wait"]: self._pending_clearoutput = True else: print("\r", file=io.stdout, end="") _imagemime = { 'image/png': 'png', 'image/jpeg': 'jpeg', 'image/svg+xml': 'svg', } def handle_rich_data(self, data): for mime in self.mime_preference: if mime in data and mime in self._imagemime: self.handle_image(data, mime) return True def handle_image(self, data, mime): handler = getattr( self, 'handle_image_{0}'.format(self.image_handler), None) if handler: handler(data, mime) def handle_image_PIL(self, data, mime): if mime not in ('image/png', 'image/jpeg'): return import PIL.Image raw = base64.decodestring(data[mime].encode('ascii')) img = PIL.Image.open(BytesIO(raw)) img.show() def handle_image_stream(self, data, mime): raw = base64.decodestring(data[mime].encode('ascii')) imageformat = self._imagemime[mime] fmt = dict(format=imageformat) args = [s.format(**fmt) for s in self.stream_image_handler] with open(os.devnull, 'w') as devnull: proc = subprocess.Popen( args, stdin=subprocess.PIPE, stdout=devnull, stderr=devnull) proc.communicate(raw) def handle_image_tempfile(self, data, mime): raw = base64.decodestring(data[mime].encode('ascii')) imageformat = self._imagemime[mime] filename = 'tmp.{0}'.format(imageformat) with NamedFileInTemporaryDirectory(filename) as f, \ open(os.devnull, 'w') as devnull: f.write(raw) f.flush() fmt = dict(file=f.name, format=imageformat) args = [s.format(**fmt) for s in self.tempfile_image_handler] subprocess.call(args, stdout=devnull, stderr=devnull) def handle_image_callable(self, data, mime): self.callable_image_handler(data) def handle_input_request(self, msg_id, timeout=0.1): """ Method to capture raw_input """ req = self.client.stdin_channel.get_msg(timeout=timeout) # in case any iopub came while we were waiting: self.handle_iopub(msg_id) if msg_id == req["parent_header"].get("msg_id"): # wrap SIGINT handler real_handler = signal.getsignal(signal.SIGINT) def double_int(sig,frame): # call real handler (forwards sigint to kernel), # then raise local interrupt, stopping local raw_input real_handler(sig,frame) raise KeyboardInterrupt signal.signal(signal.SIGINT, double_int) content = req['content'] read = getpass if content.get('password', False) else input try: raw_data = read(content["prompt"]) except EOFError: # turn EOFError into EOF character raw_data = '\x04' except KeyboardInterrupt: sys.stdout.write('\n') return finally: # restore SIGINT handler signal.signal(signal.SIGINT, real_handler) # only send stdin reply if there *was not* another request # or execution finished while we were reading. if not (self.client.stdin_channel.msg_ready() or self.client.shell_channel.msg_ready()): self.client.input(raw_data) def mainloop(self, display_banner=False): while True: try: self.interact(display_banner=display_banner) #self.interact_with_readline() # XXX for testing of a readline-decoupled repl loop, call # interact_with_readline above break except KeyboardInterrupt: # this should not be necessary, but KeyboardInterrupt # handling seems rather unpredictable... self.write("\nKeyboardInterrupt in interact()\n") self.client.shutdown() def _banner1_default(self): return "IPython Console {version}\n".format(version=release.version) def compute_banner(self): super(ZMQTerminalInteractiveShell, self).compute_banner() if self.client and not self.kernel_banner: msg_id = self.client.kernel_info() while True: try: reply = self.client.get_shell_msg(timeout=1) except Empty: break else: if reply['parent_header'].get('msg_id') == msg_id: self.kernel_banner = reply['content'].get('banner', '') break self.banner += self.kernel_banner def wait_for_kernel(self, timeout=None): """method to wait for a kernel to be ready""" tic = time.time() self.client.hb_channel.unpause() while True: msg_id = self.client.kernel_info() reply = None while True: try: reply = self.client.get_shell_msg(timeout=1) except Empty: break else: if reply['parent_header'].get('msg_id') == msg_id: return True if timeout is not None \ and (time.time() - tic) > timeout \ and not self.client.hb_channel.is_beating(): # heart failed return False return True def interact(self, display_banner=None): """Closely emulate the interactive Python console.""" # batch run -> do not interact if self.exit_now: return if display_banner is None: display_banner = self.display_banner if isinstance(display_banner, string_types): self.show_banner(display_banner) elif display_banner: self.show_banner() more = False # run a non-empty no-op, so that we don't get a prompt until # we know the kernel is ready. This keeps the connection # message above the first prompt. if not self.wait_for_kernel(self.kernel_timeout): error("Kernel did not respond\n") return if self.has_readline: self.readline_startup_hook(self.pre_readline) hlen_b4_cell = self.readline.get_current_history_length() else: hlen_b4_cell = 0 # exit_now is set by a call to %Exit or %Quit, through the # ask_exit callback. while not self.exit_now: if not self.client.is_alive(): # kernel died, prompt for action or exit action = "restart" if self.manager else "wait for restart" ans = self.ask_yes_no("kernel died, %s ([y]/n)?" % action, default='y') if ans: if self.manager: self.manager.restart_kernel(True) self.wait_for_kernel(self.kernel_timeout) else: self.exit_now = True continue try: # protect prompt block from KeyboardInterrupt # when sitting on ctrl-C self.hooks.pre_prompt_hook() if more: try: prompt = self.prompt_manager.render('in2') except Exception: self.showtraceback() if self.autoindent: self.rl_do_indent = True else: try: prompt = self.separate_in + self.prompt_manager.render('in') except Exception: self.showtraceback() line = self.raw_input(prompt) if self.exit_now: # quick exit on sys.std[in|out] close break if self.autoindent: self.rl_do_indent = False except KeyboardInterrupt: #double-guard against keyboardinterrupts during kbdint handling try: self.write('\n' + self.get_exception_only()) source_raw = self.input_splitter.raw_reset() hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell) more = False except KeyboardInterrupt: pass except EOFError: if self.autoindent: self.rl_do_indent = False if self.has_readline: self.readline_startup_hook(None) self.write('\n') self.exit() except bdb.BdbQuit: warn('The Python debugger has exited with a BdbQuit exception.\n' 'Because of how pdb handles the stack, it is impossible\n' 'for IPython to properly format this particular exception.\n' 'IPython will resume normal operation.') except: # exceptions here are VERY RARE, but they can be triggered # asynchronously by signal handlers, for example. self.showtraceback() else: try: self.input_splitter.push(line) more = self.input_splitter.push_accepts_more() except SyntaxError: # Run the code directly - run_cell takes care of displaying # the exception. more = False if (self.SyntaxTB.last_syntax_error and self.autoedit_syntax): self.edit_syntax_error() if not more: source_raw = self.input_splitter.raw_reset() hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell) self.run_cell(source_raw) # Turn off the exit flag, so the mainloop can be restarted if desired self.exit_now = False def init_history(self): """Sets up the command history. """ self.history_manager = ZMQHistoryManager(client=self.client) self.configurables.append(self.history_manager)
class InteractiveShellEmbed(TerminalInteractiveShell): dummy_mode = Bool(False) exit_msg = Unicode('') embedded = CBool(True) embedded_active = CBool(True) # Like the base class display_banner is not configurable, but here it # is True by default. display_banner = CBool(True) def __init__(self, config=None, ipython_dir=None, user_ns=None, user_module=None, custom_exceptions=((), None), usage=None, banner1=None, banner2=None, display_banner=None, exit_msg=u'', user_global_ns=None): if user_global_ns is not None: warnings.warn( "user_global_ns has been replaced by user_module. The\ parameter will be ignored.", DeprecationWarning) super(InteractiveShellEmbed, self).__init__(config=config, ipython_dir=ipython_dir, user_ns=user_ns, user_module=user_module, custom_exceptions=custom_exceptions, usage=usage, banner1=banner1, banner2=banner2, display_banner=display_banner) self.exit_msg = exit_msg # don't use the ipython crash handler so that user exceptions aren't # trapped sys.excepthook = ultratb.FormattedTB(color_scheme=self.colors, mode=self.xmode, call_pdb=self.pdb) def init_sys_modules(self): pass def init_magics(self): super(InteractiveShellEmbed, self).init_magics() self.register_magics(EmbeddedMagics) def __call__(self, header='', local_ns=None, module=None, dummy=None, stack_depth=1, global_ns=None, compile_flags=None): """Activate the interactive interpreter. __call__(self,header='',local_ns=None,module=None,dummy=None) -> Start the interpreter shell with the given local and global namespaces, and optionally print a header string at startup. The shell can be globally activated/deactivated using the dummy_mode attribute. This allows you to turn off a shell used for debugging globally. However, *each* time you call the shell you can override the current state of dummy_mode with the optional keyword parameter 'dummy'. For example, if you set dummy mode on with IPShell.dummy_mode = True, you can still have a specific call work by making it as IPShell(dummy=False). """ # If the user has turned it off, go away if not self.embedded_active: return # Normal exits from interactive mode set this flag, so the shell can't # re-enter (it checks this variable at the start of interactive mode). self.exit_now = False # Allow the dummy parameter to override the global __dummy_mode if dummy or (dummy != 0 and self.dummy_mode): return if self.has_readline: self.set_readline_completer() # self.banner is auto computed if header: self.old_banner2 = self.banner2 self.banner2 = self.banner2 + '\n' + header + '\n' else: self.old_banner2 = '' # Call the embedding code with a stack depth of 1 so it can skip over # our call and get the original caller's namespaces. self.mainloop(local_ns, module, stack_depth=stack_depth, global_ns=global_ns, compile_flags=compile_flags) self.banner2 = self.old_banner2 if self.exit_msg is not None: print self.exit_msg def mainloop(self, local_ns=None, module=None, stack_depth=0, display_banner=None, global_ns=None, compile_flags=None): """Embeds IPython into a running python program. Input: - header: An optional header message can be specified. - local_ns, module: working local namespace (a dict) and module (a module or similar object). If given as None, they are automatically taken from the scope where the shell was called, so that program variables become visible. - stack_depth: specifies how many levels in the stack to go to looking for namespaces (when local_ns or module is None). This allows an intermediate caller to make sure that this function gets the namespace from the intended level in the stack. By default (0) it will get its locals and globals from the immediate caller. - compile_flags: A bit field identifying the __future__ features that are enabled, as passed to the builtin `compile` function. If given as None, they are automatically taken from the scope where the shell was called. Warning: it's possible to use this in a program which is being run by IPython itself (via %run), but some funny things will happen (a few globals get overwritten). In the future this will be cleaned up, as there is no fundamental reason why it can't work perfectly.""" if (global_ns is not None) and (module is None): class DummyMod(object): """A dummy module object for embedded IPython.""" pass warnings.warn("global_ns is deprecated, use module instead.", DeprecationWarning) module = DummyMod() module.__dict__ = global_ns # Get locals and globals from caller if ((local_ns is None or module is None or compile_flags is None) and self.default_user_namespaces): call_frame = sys._getframe(stack_depth).f_back if local_ns is None: local_ns = call_frame.f_locals if module is None: global_ns = call_frame.f_globals module = sys.modules[global_ns['__name__']] if compile_flags is None: compile_flags = (call_frame.f_code.co_flags & compilerop.PyCF_MASK) # Save original namespace and module so we can restore them after # embedding; otherwise the shell doesn't shut down correctly. orig_user_module = self.user_module orig_user_ns = self.user_ns orig_compile_flags = self.compile.flags # Update namespaces and fire up interpreter # The global one is easy, we can just throw it in if module is not None: self.user_module = module # But the user/local one is tricky: ipython needs it to store internal # data, but we also need the locals. We'll throw our hidden variables # like _ih and get_ipython() into the local namespace, but delete them # later. if local_ns is not None: self.user_ns = local_ns self.init_user_ns() # Compiler flags if compile_flags is not None: self.compile.flags = compile_flags # Patch for global embedding to make sure that things don't overwrite # user globals accidentally. Thanks to Richard <*****@*****.**> # FIXME. Test this a bit more carefully (the if.. is new) # N.B. This can't now ever be called. Not sure what it was for. # And now, since it wasn't called in the previous version, I'm # commenting out these lines so they can't be called with my new changes # --TK, 2011-12-10 #if local_ns is None and module is None: # self.user_global_ns.update(__main__.__dict__) # make sure the tab-completer has the correct frame information, so it # actually completes using the frame's locals/globals self.set_completer_frame() with nested(self.builtin_trap, self.display_trap): self.interact(display_banner=display_banner) # now, purge out the local namespace of IPython's hidden variables. if local_ns is not None: for name in self.user_ns_hidden: local_ns.pop(name, None) # Restore original namespace so shell can shut down when we exit. self.user_module = orig_user_module self.user_ns = orig_user_ns self.compile.flags = orig_compile_flags
class KernelManager(LoggingConfigurable, ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): return zmq.Context.instance() # The Session to use for communication with the kernel. session = Instance(Session) def _session_default(self): return Session(parent=self) # the class to create with our `client` method client_class = DottedObjectName( 'IPython.kernel.blocking.BlockingKernelClient') client_factory = Type() def _client_class_changed(self, name, old, new): self.client_factory = import_item(str(new)) # The kernel process with which the KernelManager is communicating. # generally a Popen instance kernel = Any() kernel_spec_manager = Instance(kernelspec.KernelSpecManager) def _kernel_spec_manager_default(self): return kernelspec.KernelSpecManager(ipython_dir=self.ipython_dir) kernel_name = Unicode('python') kernel_spec = Instance(kernelspec.KernelSpec) def _kernel_spec_default(self): return self.kernel_spec_manager.get_kernel_spec(self.kernel_name) def _kernel_name_changed(self, name, old, new): self.kernel_spec = self.kernel_spec_manager.get_kernel_spec(new) self.ipython_kernel = new in {'python', 'python2', 'python3'} kernel_cmd = List(Unicode, config=True, help="""DEPRECATED: Use kernel_name instead. The Popen Command to launch the kernel. Override this if you have a custom kernel. If kernel_cmd is specified in a configuration file, IPython does not pass any arguments to the kernel, because it cannot make any assumptions about the arguments that the kernel understands. In particular, this means that the kernel does not receive the option --debug if it given on the IPython command line. """) def _kernel_cmd_changed(self, name, old, new): warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " "start different kernels.") self.ipython_kernel = False ipython_kernel = Bool(True) ipython_dir = Unicode() def _ipython_dir_default(self): return get_ipython_dir() # Protected traits _launch_args = Any() _control_socket = Any() _restarter = Any() autorestart = Bool(False, config=True, help="""Should we autorestart the kernel if it dies.""") def __del__(self): self._close_control_socket() self.cleanup_connection_file() #-------------------------------------------------------------------------- # Kernel restarter #-------------------------------------------------------------------------- def start_restarter(self): pass def stop_restarter(self): pass def add_restart_callback(self, callback, event='restart'): """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback, event='restart'): """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) #-------------------------------------------------------------------------- # create a Client connected to our Kernel #-------------------------------------------------------------------------- def client(self, **kwargs): """Create a client configured to connect to our kernel""" if self.client_factory is None: self.client_factory = import_item(self.client_class) kw = {} kw.update(self.get_connection_info()) kw.update( dict( connection_file=self.connection_file, session=self.session, parent=self, )) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) #-------------------------------------------------------------------------- # Kernel management #-------------------------------------------------------------------------- def format_kernel_cmd(self, **kw): """replace templated args (e.g. {connection_file})""" if self.kernel_cmd: cmd = self.kernel_cmd elif self.kernel_name == 'python': # The native kernel gets special handling cmd = make_ipkernel_cmd( 'from IPython.kernel.zmq.kernelapp import main; main()', **kw) else: cmd = self.kernel_spec.argv ns = dict(connection_file=self.connection_file) ns.update(self._launch_args) pat = re.compile(r'\{([A-Za-z0-9_]+)\}') def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [pat.sub(from_ns, arg) for arg in cmd] def _launch_kernel(self, kernel_cmd, **kw): """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ return launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown def _connect_control_socket(self): if self._control_socket is None: self._control_socket = self.connect_control() self._control_socket.linger = 100 def _close_control_socket(self): if self._control_socket is None: return self._control_socket.close() self._control_socket = None def start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- **kw : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError( "Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % local_ips()) # write connection file / get default ports self.write_connection_file() # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd kernel_cmd = self.format_kernel_cmd(**kw) if self.kernel_cmd: # If kernel_cmd has been set manually, don't refer to a kernel spec env = os.environ else: # Environment variables from kernel spec are added to os.environ env = os.environ.copy() env.update(self.kernel_spec.env or {}) # launch the kernel subprocess self.kernel = self._launch_kernel(kernel_cmd, env=env, ipython_kernel=self.ipython_kernel, **kw) self.start_restarter() self._connect_control_socket() def _send_shutdown_request(self, restart=False): """TODO: send a shutdown request via control channel""" content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) self.session.send(self._control_socket, msg) def shutdown_kernel(self, now=False, restart=False): """Attempts to the stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the shell channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ # Stop monitoring for restarting while we shutdown. self.stop_restarter() # FIXME: Shutdown does not work on Windows due to ZMQ errors! if now or sys.platform == 'win32': if self.has_kernel: self._kill_kernel() else: # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. self._send_shutdown_request(restart=restart) for i in range(10): if self.is_alive(): time.sleep(0.1) else: break else: # OK, we've waited long enough. if self.has_kernel: self._kill_kernel() if not restart: self.cleanup_connection_file() self.cleanup_ipc_files() else: self.cleanup_ipc_files() self._close_control_socket() def restart_kernel(self, now=False, **kw): """Restarts a kernel with the arguments that were used to launch it. If the old kernel was launched with random ports, the same ports will be used for the new kernel. The same connection file is used again. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. **kw : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. self.shutdown_kernel(now=now, restart=True) # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) # FIXME: Messages get dropped in Windows due to probable ZMQ bug # unless there is some delay here. if sys.platform == 'win32': time.sleep(0.2) @property def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None def _kill_kernel(self): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Block until the kernel terminates. self.kernel.wait() self.kernel = None else: raise RuntimeError("Cannot kill kernel. No kernel is running!") def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: if sys.platform == 'win32': from .zmq.parentpoller import ParentPollerWindows as Poller Poller.send_interrupt(self.kernel.win32_interrupt_event) else: self.kernel.send_signal(signal.SIGINT) else: raise RuntimeError( "Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): """Sends a signal to the kernel. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False
class Session(Configurable): """Object for handling serialization and sending of messages. The Session object handles building messages and sending them with ZMQ sockets or ZMQStream objects. Objects can communicate with each other over the network via Session objects, and only need to work with the dict-based IPython message spec. The Session will handle serialization/deserialization, security, and metadata. Sessions support configurable serialization via packer/unpacker traits, and signing with HMAC digests via the key/keyfile traits. Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : bytes the ID of this Session object. The default is to generate a new UUID. username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ debug=Bool(False, config=True, help="""Debug output in the Session""") packer = DottedObjectName('json',config=True, help="""The name of the packer for serializing messages. Should be one of 'json', 'pickle', or an import name for a custom callable serializer.""") def _packer_changed(self, name, old, new): if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.unpacker = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.unpacker = new else: self.pack = import_item(str(new)) unpacker = DottedObjectName('json', config=True, help="""The name of the unpacker for unserializing messages. Only used with custom functions for `packer`.""") def _unpacker_changed(self, name, old, new): if new.lower() == 'json': self.pack = json_packer self.unpack = json_unpacker self.packer = new elif new.lower() == 'pickle': self.pack = pickle_packer self.unpack = pickle_unpacker self.packer = new else: self.unpack = import_item(str(new)) session = CUnicode(u'', config=True, help="""The UUID identifying this session.""") def _session_default(self): u = unicode_type(uuid.uuid4()) self.bsession = u.encode('ascii') return u def _session_changed(self, name, old, new): self.bsession = self.session.encode('ascii') # bsession is the session as bytes bsession = CBytes(b'') username = Unicode(str_to_unicode(os.environ.get('USER', 'username')), help="""Username for the Session. Default is your system username.""", config=True) metadata = Dict({}, config=True, help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""") # if 0, no adapting to do. adapt_version = Integer(0) # message signature related traits: key = CBytes(config=True, help="""execution key, for signing messages.""") def _key_default(self): return str_to_bytes(str(uuid.uuid4())) def _key_changed(self): self._new_auth() signature_scheme = Unicode('hmac-sha256', config=True, help="""The digest scheme used to construct the message signatures. Must have the form 'hmac-HASH'.""") def _signature_scheme_changed(self, name, old, new): if not new.startswith('hmac-'): raise TraitError("signature_scheme must start with 'hmac-', got %r" % new) hash_name = new.split('-', 1)[1] try: self.digest_mod = getattr(hashlib, hash_name) except AttributeError: raise TraitError("hashlib has no such attribute: %s" % hash_name) self._new_auth() digest_mod = Any() def _digest_mod_default(self): return hashlib.sha256 auth = Instance(hmac.HMAC, allow_none=True) def _new_auth(self): if self.key: self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod) else: self.auth = None digest_history = Set() digest_history_size = Integer(2**16, config=True, help="""The maximum number of digests to remember. The digest history will be culled when it exceeds this value. """ ) keyfile = Unicode('', config=True, help="""path to file containing execution key.""") def _keyfile_changed(self, name, old, new): with open(new, 'rb') as f: self.key = f.read().strip() # for protecting against sends from forks pid = Integer() # serialization traits: pack = Any(default_packer) # the actual packer function def _pack_changed(self, name, old, new): if not callable(new): raise TypeError("packer must be callable, not %s"%type(new)) unpack = Any(default_unpacker) # the actual packer function def _unpack_changed(self, name, old, new): # unpacker is not checked - it is assumed to be if not callable(new): raise TypeError("unpacker must be callable, not %s"%type(new)) # thresholds: copy_threshold = Integer(2**16, config=True, help="Threshold (in bytes) beyond which a buffer should be sent without copying.") buffer_threshold = Integer(MAX_BYTES, config=True, help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.") item_threshold = Integer(MAX_ITEMS, config=True, help="""The maximum number of items for a container to be introspected for custom serialization. Containers larger than this are pickled outright. """ ) def __init__(self, **kwargs): """create a Session object Parameters ---------- debug : bool whether to trigger extra debugging statements packer/unpacker : str : 'json', 'pickle' or import_string importstrings for methods to serialize message parts. If just 'json' or 'pickle', predefined JSON and pickle packers will be used. Otherwise, the entire importstring must be used. The functions must accept at least valid JSON input, and output *bytes*. For example, to use msgpack: packer = 'msgpack.packb', unpacker='msgpack.unpackb' pack/unpack : callables You can also set the pack/unpack callables for serialization directly. session : unicode (must be ascii) the ID of this Session object. The default is to generate a new UUID. bsession : bytes The session as bytes username : unicode username added to message headers. The default is to ask the OS. key : bytes The key used to initialize an HMAC signature. If unset, messages will not be signed or checked. signature_scheme : str The message digest scheme. Currently must be of the form 'hmac-HASH', where 'HASH' is a hashing function available in Python's hashlib. The default is 'hmac-sha256'. This is ignored if 'key' is empty. keyfile : filepath The file containing a key. If this is set, `key` will be initialized to the contents of the file. """ super(Session, self).__init__(**kwargs) self._check_packers() self.none = self.pack({}) # ensure self._session_default() if necessary, so bsession is defined: self.session self.pid = os.getpid() self._new_auth() @property def msg_id(self): """always return new uuid""" return str(uuid.uuid4()) def _check_packers(self): """check packers for datetime support.""" pack = self.pack unpack = self.unpack # check simple serialization msg = dict(a=[1,'hi']) try: packed = pack(msg) except Exception as e: msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg) ) # ensure packed message is bytes if not isinstance(packed, bytes): raise ValueError("message packed to %r, but bytes are required"%type(packed)) # check that unpack is pack's inverse try: unpacked = unpack(packed) assert unpacked == msg except Exception as e: msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}" if self.packer == 'json': jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod else: jsonmsg = "" raise ValueError( msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg) ) # check datetime support msg = dict(t=datetime.now()) try: unpacked = unpack(pack(msg)) if isinstance(unpacked['t'], datetime): raise ValueError("Shouldn't deserialize to datetime") except Exception: self.pack = lambda o: pack(squash_dates(o)) self.unpack = lambda s: unpack(s) def msg_header(self, msg_type): return msg_header(self.msg_id, msg_type, self.username, self.session) def msg(self, msg_type, content=None, parent=None, header=None, metadata=None): """Return the nested message dict. This format is different from what is sent over the wire. The serialize/deserialize methods converts this nested message dict to the wire format, which is a list of message parts. """ msg = {} header = self.msg_header(msg_type) if header is None else header msg['header'] = header msg['msg_id'] = header['msg_id'] msg['msg_type'] = header['msg_type'] msg['parent_header'] = {} if parent is None else extract_header(parent) msg['content'] = {} if content is None else content msg['metadata'] = self.metadata.copy() if metadata is not None: msg['metadata'].update(metadata) return msg def sign(self, msg_list): """Sign a message with HMAC digest. If no auth, return b''. Parameters ---------- msg_list : list The [p_header,p_parent,p_content] part of the message list. """ if self.auth is None: return b'' h = self.auth.copy() for m in msg_list: h.update(m) return str_to_bytes(h.hexdigest()) def serialize(self, msg, ident=None): """Serialize the message components to bytes. This is roughly the inverse of deserialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg : dict or Message The next message dict as returned by the self.msg method. Returns ------- msg_list : list The list of bytes objects to be sent with the format:: [ident1, ident2, ..., DELIM, HMAC, p_header, p_parent, p_metadata, p_content, buffer1, buffer2, ...] In this list, the ``p_*`` entities are the packed or serialized versions, so if JSON is used, these are utf8 encoded JSON strings. """ content = msg.get('content', {}) if content is None: content = self.none elif isinstance(content, dict): content = self.pack(content) elif isinstance(content, bytes): # content is already packed, as in a relayed message pass elif isinstance(content, unicode_type): # should be bytes, but JSON often spits out unicode content = content.encode('utf8') else: raise TypeError("Content incorrect type: %s"%type(content)) real_message = [self.pack(msg['header']), self.pack(msg['parent_header']), self.pack(msg['metadata']), content, ] to_send = [] if isinstance(ident, list): # accept list of idents to_send.extend(ident) elif ident is not None: to_send.append(ident) to_send.append(DELIM) signature = self.sign(real_message) to_send.append(signature) to_send.extend(real_message) return to_send def send(self, stream, msg_or_type, content=None, parent=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Build and send a message via stream or socket. The message format used by this function internally is as follows: [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content, buffer1,buffer2,...] The serialize/deserialize methods convert the nested message dict into this format. Parameters ---------- stream : zmq.Socket or ZMQStream The socket-like object used to send the data. msg_or_type : str or Message/dict Normally, msg_or_type will be a msg_type unless a message is being sent more than once. If a header is supplied, this can be set to None and the msg_type will be pulled from the header. content : dict or None The content of the message (ignored if msg_or_type is a message). header : dict or None The header dict for the message (ignored if msg_to_type is a message). parent : Message or dict or None The parent or parent header describing the parent of this message (ignored if msg_or_type is a message). ident : bytes or list of bytes The zmq.IDENTITY routing path. metadata : dict or None The metadata describing the message buffers : list or None The already-serialized buffers to be appended to the message. track : bool Whether to track. Only for use with Sockets, because ZMQStream objects cannot track messages. Returns ------- msg : dict The constructed message. """ if not isinstance(stream, zmq.Socket): # ZMQStreams and dummy sockets do not support tracking. track = False if isinstance(msg_or_type, (Message, dict)): # We got a Message or message dict, not a msg_type so don't # build a new Message. msg = msg_or_type buffers = buffers or msg.get('buffers', []) else: msg = self.msg(msg_or_type, content=content, parent=parent, header=header, metadata=metadata) if not os.getpid() == self.pid: get_logger().warn("WARNING: attempted to send message from fork\n%s", msg ) return buffers = [] if buffers is None else buffers if self.adapt_version: msg = adapt(msg, self.adapt_version) to_send = self.serialize(msg, ident) to_send.extend(buffers) longest = max([ len(s) for s in to_send ]) copy = (longest < self.copy_threshold) if buffers and track and not copy: # only really track when we are doing zero-copy buffers tracker = stream.send_multipart(to_send, copy=False, track=True) else: # use dummy tracker, which will be done immediately tracker = DONE stream.send_multipart(to_send, copy=copy) if self.debug: pprint.pprint(msg) pprint.pprint(to_send) pprint.pprint(buffers) msg['tracker'] = tracker return msg def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None): """Send a raw message via ident path. This method is used to send a already serialized message. Parameters ---------- stream : ZMQStream or Socket The ZMQ stream or socket to use for sending the message. msg_list : list The serialized list of messages to send. This only includes the [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of the message. ident : ident or list A single ident or a list of idents to use in sending. """ to_send = [] if isinstance(ident, bytes): ident = [ident] if ident is not None: to_send.extend(ident) to_send.append(DELIM) to_send.append(self.sign(msg_list)) to_send.extend(msg_list) stream.send_multipart(to_send, flags, copy=copy) def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): """Receive and unpack a message. Parameters ---------- socket : ZMQStream or Socket The socket or stream to use in receiving. Returns ------- [idents], msg [idents] is a list of idents and msg is a nested message dict of same format as self.msg returns. """ if isinstance(socket, ZMQStream): socket = socket.socket try: msg_list = socket.recv_multipart(mode, copy=copy) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: # We can convert EAGAIN to None as we know in this case # recv_multipart won't return None. return None,None else: raise # split multipart message into identity list and message dict # invalid large messages can cause very expensive string comparisons idents, msg_list = self.feed_identities(msg_list, copy) try: return idents, self.deserialize(msg_list, content=content, copy=copy) except Exception as e: # TODO: handle it raise e def feed_identities(self, msg_list, copy=True): """Split the identities from the rest of the message. Feed until DELIM is reached, then return the prefix as idents and remainder as msg_list. This is easily broken by setting an IDENT to DELIM, but that would be silly. Parameters ---------- msg_list : a list of Message or bytes objects The message to be split. copy : bool flag determining whether the arguments are bytes or Messages Returns ------- (idents, msg_list) : two lists idents will always be a list of bytes, each of which is a ZMQ identity. msg_list will be a list of bytes or zmq.Messages of the form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and should be unpackable/unserializable via self.deserialize at this point. """ if copy: idx = msg_list.index(DELIM) return msg_list[:idx], msg_list[idx+1:] else: failed = True for idx,m in enumerate(msg_list): if m.bytes == DELIM: failed = False break if failed: raise ValueError("DELIM not in msg_list") idents, msg_list = msg_list[:idx], msg_list[idx+1:] return [m.bytes for m in idents], msg_list def _add_digest(self, signature): """add a digest to history to protect against replay attacks""" if self.digest_history_size == 0: # no history, never add digests return self.digest_history.add(signature) if len(self.digest_history) > self.digest_history_size: # threshold reached, cull 10% self._cull_digest_history() def _cull_digest_history(self): """cull the digest history Removes a randomly selected 10% of the digest history """ current = len(self.digest_history) n_to_cull = max(int(current // 10), current - self.digest_history_size) if n_to_cull >= current: self.digest_history = set() return to_cull = random.sample(self.digest_history, n_to_cull) self.digest_history.difference_update(to_cull) def deserialize(self, msg_list, content=True, copy=True): """Unserialize a msg_list to a nested message dict. This is roughly the inverse of serialize. The serialize/deserialize methods work with full message lists, whereas pack/unpack work with the individual message parts in the message list. Parameters ---------- msg_list : list of bytes or Message objects The list of message parts of the form [HMAC,p_header,p_parent, p_metadata,p_content,buffer1,buffer2,...]. content : bool (True) Whether to unpack the content dict (True), or leave it packed (False). copy : bool (True) Whether msg_list contains bytes (True) or the non-copying Message objects in each place (False). Returns ------- msg : dict The nested message dict with top-level keys [header, parent_header, content, buffers]. The buffers are returned as memoryviews. """ minlen = 5 message = {} if not copy: # pyzmq didn't copy the first parts of the message, so we'll do it for i in range(minlen): msg_list[i] = msg_list[i].bytes if self.auth is not None: signature = msg_list[0] if not signature: raise ValueError("Unsigned Message") if signature in self.digest_history: raise ValueError("Duplicate Signature: %r" % signature) self._add_digest(signature) check = self.sign(msg_list[1:5]) if not compare_digest(signature, check): raise ValueError("Invalid Signature: %r" % signature) if not len(msg_list) >= minlen: raise TypeError("malformed message, must have at least %i elements"%minlen) header = self.unpack(msg_list[1]) message['header'] = extract_dates(header) message['msg_id'] = header['msg_id'] message['msg_type'] = header['msg_type'] message['parent_header'] = extract_dates(self.unpack(msg_list[2])) message['metadata'] = self.unpack(msg_list[3]) if content: message['content'] = self.unpack(msg_list[4]) else: message['content'] = msg_list[4] buffers = [memoryview(b) for b in msg_list[5:]] if buffers and buffers[0].shape is None: # force copy to workaround pyzmq #646 buffers = [memoryview(b.bytes) for b in msg_list[5:]] message['buffers'] = buffers # adapt to the current version return adapt(message) def unserialize(self, *args, **kwargs): warnings.warn( "Session.unserialize is deprecated. Use Session.deserialize.", DeprecationWarning, ) return self.deserialize(*args, **kwargs)
class Kernel(Configurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) def _eventloop_changed(self, name, old, new): """schedule call to eventloop from IOLoop""" loop = ioloop.IOLoop.instance() loop.add_callback(self.enter_eventloop) session = Instance(Session) profile_dir = Instance('IPython.core.profiledir.ProfileDir') shell_streams = List() control_stream = Instance(ZMQStream) iopub_socket = Instance(zmq.Socket) stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) # identities: int_id = Integer(-1) ident = Unicode() def _ident_default(self): return unicode_type(uuid.uuid4()) # Private interface _darwin_app_nap = Bool( True, config=True, help="""Whether to use appnope for compatiblity with OS X App Nap. Only affects OS X >= 10.9. """) # track associations with current request _allow_stdin = Bool(False) _parent_header = Dict() _parent_ident = Any(b'') # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005, config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05, config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() # set of aborted msg_ids aborted = Set() # Track execution count here. For IPython, we override this to use the # execution count we store in the shell. execution_count = 0 def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'inspect_request', 'history_request', 'kernel_info_request', 'connect_request', 'shutdown_request', 'apply_request', ] self.shell_handlers = {} for msg_type in msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) control_msg_types = msg_types + ['clear_request', 'abort_request'] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) def dispatch_control(self, msg): """dispatch control requests""" idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Control Message", exc_info=True) return self.log.debug("Control received: %s", msg) # Set the parent message for side effects. self.set_parent(idents, msg) self._publish_status(u'busy') header = msg['header'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: try: handler(self.control_stream, idents, msg) except Exception: self.log.error("Exception in control handler:", exc_info=True) sys.stdout.flush() sys.stderr.flush() self._publish_status(u'idle') def dispatch_shell(self, stream, msg): """dispatch shell requests""" # flush control requests first if self.control_stream: self.control_stream.flush() idents, msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return # Set the parent message for side effects. self.set_parent(idents, msg) self._publish_status(u'busy') header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) if msg_id in self.aborted: self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: # ensure default_int_handler during handler call sig = signal(SIGINT, default_int_handler) self.log.debug("%s: %s", msg_type, msg) try: handler(stream, idents, msg) except Exception: self.log.error("Exception in message handler:", exc_info=True) finally: signal(SIGINT, sig) sys.stdout.flush() sys.stderr.flush() self._publish_status(u'idle') def enter_eventloop(self): """enter eventloop""" self.log.info("entering eventloop %s", self.eventloop) for stream in self.shell_streams: # flush any pending replies, # which may be skipped by entering the eventloop stream.flush(zmq.POLLOUT) # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel self.log.error("KeyboardInterrupt caught in kernel") continue else: # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break self.log.info("exiting eventloop") def start(self): """register dispatchers for streams""" if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_shell(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) # publish idle status self._publish_status('starting') def do_one_iteration(self): """step eventloop just once""" if self.control_stream: self.control_stream.flush() for stream in self.shell_streams: # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _make_metadata(self, other=None): """init metadata dict, for execute/apply_reply""" new_md = { 'dependencies_met': True, 'engine': self.ident, 'started': datetime.now(), } if other: new_md.update(other) return new_md def _publish_execute_input(self, code, parent, execution_count): """Publish the code request on the iopub stream.""" self.session.send(self.iopub_socket, u'execute_input', { u'code': code, u'execution_count': execution_count }, parent=parent, ident=self._topic('execute_input')) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send( self.iopub_socket, u'status', {u'execution_state': status}, parent=parent or self._parent_header, ident=self._topic('status'), ) def set_parent(self, ident, parent): """Set the current parent_header Side effects (IOPub messages) and replies are associated with the request that caused them via the parent_header. The parent identity is used to route input_request messages on the stdin channel. """ self._parent_ident = ident self._parent_header = parent def send_response(self, stream, msg_or_type, content=None, ident=None, buffers=None, track=False, header=None, metadata=None): """Send a response to the message we're currently processing. This accepts all the parameters of :meth:`IPython.kernel.zmq.session.Session.send` except ``parent``. This relies on :meth:`set_parent` having been called for the current message. """ return self.session.send(stream, msg_or_type, content, self._parent_header, ident, buffers, track, header, metadata) def execute_request(self, stream, ident, parent): """handle an execute_request""" try: content = parent[u'content'] code = py3compat.cast_unicode_py2(content[u'code']) silent = content[u'silent'] store_history = content.get(u'store_history', not silent) user_expressions = content.get('user_expressions', {}) allow_stdin = content.get('allow_stdin', False) except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return md = self._make_metadata(parent['metadata']) # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self.execution_count += 1 self._publish_execute_input(code, parent, self.execution_count) reply_content = self.do_execute(code, silent, store_history, user_expressions, allow_stdin) # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) md['status'] = reply_content['status'] if reply_content['status'] == 'error' and \ reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=md, ident=ident) self.log.debug("%s", reply_msg) if not silent and reply_msg['content']['status'] == u'error': self._abort_queues() def do_execute(self, code, silent, store_history=True, user_experssions=None, allow_stdin=False): """Execute user code. Must be overridden by subclasses. """ raise NotImplementedError def complete_request(self, stream, ident, parent): content = parent['content'] code = content['code'] cursor_pos = content['cursor_pos'] matches = self.do_complete(code, cursor_pos) matches = json_clean(matches) completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) def do_complete(self, code, cursor_pos): """Override in subclasses to find completions. """ return { 'matches': [], 'cursor_end': cursor_pos, 'cursor_start': cursor_pos, 'metadata': {}, 'status': 'ok' } def inspect_request(self, stream, ident, parent): content = parent['content'] reply_content = self.do_inspect(content['code'], content['cursor_pos'], content.get('detail_level', 0)) # Before we send this object over, we scrub it for JSON usage reply_content = json_clean(reply_content) msg = self.session.send(stream, 'inspect_reply', reply_content, parent, ident) self.log.debug("%s", msg) def do_inspect(self, code, cursor_pos, detail_level=0): """Override in subclasses to allow introspection. """ return {'status': 'ok', 'data': {}, 'metadata': {}, 'found': False} def history_request(self, stream, ident, parent): content = parent['content'] reply_content = self.do_history(**content) reply_content = json_clean(reply_content) msg = self.session.send(stream, 'history_reply', reply_content, parent, ident) self.log.debug("%s", msg) def do_history(self, hist_access_type, output, raw, session=None, start=None, stop=None, n=None, pattern=None, unique=False): """Override in subclasses to access history. """ return {'history': []} def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) @property def kernel_info(self): return { 'protocol_version': release.kernel_protocol_version, 'implementation': self.implementation, 'implementation_version': self.implementation_version, 'language': self.language, 'language_version': self.language_version, 'banner': self.banner, } def kernel_info_request(self, stream, ident, parent): msg = self.session.send(stream, 'kernel_info_reply', self.kernel_info, parent, ident) self.log.debug("%s", msg) def shutdown_request(self, stream, ident, parent): content = self.do_shutdown(parent['content']['restart']) self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', content, parent) self._at_shutdown() # call sys.exit after a short delay loop = ioloop.IOLoop.instance() loop.add_timeout(time.time() + 0.1, loop.stop) def do_shutdown(self, restart): """Override in subclasses to do things when the frontend shuts down the kernel. """ return {'status': 'ok', 'restart': restart} #--------------------------------------------------------------------------- # Engine methods #--------------------------------------------------------------------------- def apply_request(self, stream, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] except: self.log.error("Got bad msg: %s", parent, exc_info=True) return md = self._make_metadata(parent['metadata']) reply_content, result_buf = self.do_apply(content, bufs, msg_id, md) # put 'ok'/'error' status in header, for scheduler introspection: md['status'] = reply_content['status'] # flush i/o sys.stdout.flush() sys.stderr.flush() self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident, buffers=result_buf, metadata=md) def do_apply(self, content, bufs, msg_id, reply_metadata): """Override in subclasses to support the IPython parallel framework. """ raise NotImplementedError #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- def abort_request(self, stream, ident, parent): """abort a specific msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, string_types): msg_ids = [msg_ids] if not msg_ids: self._abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" content = self.do_clear() self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content=content) def do_clear(self): """Override in subclasses to clear the namespace This is only required for IPython.parallel. """ raise NotImplementedError #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _topic(self, topic): """prefixed topic for IOPub messages""" if self.int_id >= 0: base = "engine.%i" % self.int_id else: base = "kernel.%s" % self.ident return py3compat.cast_bytes("%s.%s" % (base, topic)) def _abort_queues(self): for stream in self.shell_streams: if stream: self._abort_queue(stream) def _abort_queue(self, stream): poller = zmq.Poller() poller.register(stream.socket, zmq.POLLIN) while True: idents, msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' status = {'status': 'aborted'} md = {'engine': self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. poller.poll(50) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this " "frontend does not support stdin.") def getpass(self, prompt=''): """Forward getpass to frontends Raises ------ StdinNotImplentedError if active frontend doesn't support stdin. """ if not self._allow_stdin: raise StdinNotImplementedError( "getpass was called, but this frontend does not support input requests." ) return self._input_request( prompt, self._parent_ident, self._parent_header, password=True, ) def raw_input(self, prompt=''): """Forward raw_input to frontends Raises ------ StdinNotImplentedError if active frontend doesn't support stdin. """ if not self._allow_stdin: raise StdinNotImplementedError( "raw_input was called, but this frontend does not support input requests." ) return self._input_request( prompt, self._parent_ident, self._parent_header, password=False, ) def _input_request(self, prompt, ident, parent, password=False): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # flush the stdin socket, to purge stale replies while True: try: self.stdin_socket.recv_multipart(zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: break else: raise # Send the input request. content = json_clean(dict(prompt=prompt, password=password)) self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) except KeyboardInterrupt: # re-raise KeyboardInterrupt, to truncate traceback raise KeyboardInterrupt else: break try: value = py3compat.unicode_to_str(reply['content']['value']) except: self.log.error("Bad input_reply: %s", parent) value = '' if value == '\x04': # EOF raise EOFError return value def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [s.flush(zmq.POLLOUT) for s in self.shell_streams]
class InspectReply(MimeBundle): found = Bool()
class IPControllerApp(BaseParallelApplication): name = u'ipcontroller' description = _description examples = _examples config_file_name = Unicode(default_config_file_name) classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB] + real_dbs # change default to True auto_create = Bool(True, config=True, help="""Whether to create profile dir if it doesn't exist.""") reuse_files = Bool(False, config=True, help="""Whether to reuse existing json connection files. If False, connection files will be removed on a clean exit. """ ) restore_engines = Bool(False, config=True, help="""Reload engine state from JSON file """ ) ssh_server = Unicode(u'', config=True, help="""ssh url for clients to use when connecting to the Controller processes. It should be of the form: [user@]server[:port]. The Controller's listening addresses must be accessible from the ssh server""", ) engine_ssh_server = Unicode(u'', config=True, help="""ssh url for engines to use when connecting to the Controller processes. It should be of the form: [user@]server[:port]. The Controller's listening addresses must be accessible from the ssh server""", ) location = Unicode(u'', config=True, help="""The external IP or domain name of the Controller, used for disambiguating engine and client connections.""", ) import_statements = List([], config=True, help="import statements to be run at startup. Necessary in some environments" ) use_threads = Bool(False, config=True, help='Use threads instead of processes for the schedulers', ) engine_json_file = Unicode('ipcontroller-engine.json', config=True, help="JSON filename where engine connection info will be stored.") client_json_file = Unicode('ipcontroller-client.json', config=True, help="JSON filename where client connection info will be stored.") def _cluster_id_changed(self, name, old, new): super(IPControllerApp, self)._cluster_id_changed(name, old, new) self.engine_json_file = "%s-engine.json" % self.name self.client_json_file = "%s-client.json" % self.name # internal children = List() mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') def _use_threads_changed(self, name, old, new): self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') write_connection_files = Bool(True, help="""Whether to write connection files to disk. True in all cases other than runs with `reuse_files=True` *after the first* """ ) aliases = Dict(aliases) flags = Dict(flags) def save_connection_dict(self, fname, cdict): """save a connection dict to json file.""" c = self.config url = cdict['registration'] location = cdict['location'] if not location: if PUBLIC_IPS: location = PUBLIC_IPS[-1] else: self.log.warn("Could not identify this machine's IP, assuming %s." " You may need to specify '--location=<external_ip_address>' to help" " IPython decide when to connect via loopback." % LOCALHOST) location = LOCALHOST cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) self.log.info("writing connection info to %s", fname) with open(fname, 'w') as f: f.write(json.dumps(cdict, indent=2)) os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) def load_config_from_json(self): """load config from existing json connector files.""" c = self.config self.log.debug("loading config from JSON") # load engine config fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file) self.log.info("loading connection info from %s", fname) with open(fname) as f: ecfg = json.loads(f.read()) # json gives unicode, Session.key wants bytes c.Session.key = ecfg['exec_key'].encode('ascii') xport,ip = ecfg['interface'].split('://') c.HubFactory.engine_ip = ip c.HubFactory.engine_transport = xport self.location = ecfg['location'] if not self.engine_ssh_server: self.engine_ssh_server = ecfg['ssh'] # load client config fname = os.path.join(self.profile_dir.security_dir, self.client_json_file) self.log.info("loading connection info from %s", fname) with open(fname) as f: ccfg = json.loads(f.read()) for key in ('exec_key', 'registration', 'pack', 'unpack'): assert ccfg[key] == ecfg[key], "mismatch between engine and client info: %r" % key xport,addr = ccfg['interface'].split('://') c.HubFactory.client_transport = xport c.HubFactory.client_ip = ip if not self.ssh_server: self.ssh_server = ccfg['ssh'] # load port config: c.HubFactory.regport = ecfg['registration'] c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong']) c.HubFactory.control = (ccfg['control'], ecfg['control']) c.HubFactory.mux = (ccfg['mux'], ecfg['mux']) c.HubFactory.task = (ccfg['task'], ecfg['task']) c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub']) c.HubFactory.notifier_port = ccfg['notification'] def cleanup_connection_files(self): if self.reuse_files: self.log.debug("leaving JSON connection files for reuse") return self.log.debug("cleaning up JSON connection files") for f in (self.client_json_file, self.engine_json_file): f = os.path.join(self.profile_dir.security_dir, f) try: os.remove(f) except Exception as e: self.log.error("Failed to cleanup connection file: %s", e) else: self.log.debug(u"removed %s", f) def load_secondary_config(self): """secondary config, loading from JSON and setting defaults""" if self.reuse_files: try: self.load_config_from_json() except (AssertionError,IOError) as e: self.log.error("Could not load config from JSON: %s" % e) else: # successfully loaded config from JSON, and reuse=True # no need to wite back the same file self.write_connection_files = False # switch Session.key default to secure default_secure(self.config) self.log.debug("Config changed") self.log.debug(repr(self.config)) def init_hub(self): c = self.config self.do_import_statements() try: self.factory = HubFactory(config=c, log=self.log) # self.start_logging() self.factory.init_hub() except TraitError: raise except Exception: self.log.error("Couldn't construct the Controller", exc_info=True) self.exit(1) if self.write_connection_files: # save to new json config files f = self.factory base = { 'exec_key' : f.session.key.decode('ascii'), 'location' : self.location, 'pack' : f.session.packer, 'unpack' : f.session.unpacker, } cdict = {'ssh' : self.ssh_server} cdict.update(f.client_info) cdict.update(base) self.save_connection_dict(self.client_json_file, cdict) edict = {'ssh' : self.engine_ssh_server} edict.update(f.engine_info) edict.update(base) self.save_connection_dict(self.engine_json_file, edict) fname = "engines%s.json" % self.cluster_id self.factory.hub.engine_state_file = os.path.join(self.profile_dir.log_dir, fname) if self.restore_engines: self.factory.hub._load_engine_state() def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) f = self.factory ident = f.session.bsession # disambiguate url, in case of * monitor_url = disambiguate_url(f.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') q.bind_in(f.client_url('iopub')) q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") q.bind_out(f.engine_url('iopub')) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(monitor_url) q.daemon=True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') q.bind_in(f.client_url('mux')) q.setsockopt_in(zmq.IDENTITY, b'mux_in') q.bind_out(f.engine_url('mux')) q.setsockopt_out(zmq.IDENTITY, b'mux_out') q.connect_mon(monitor_url) q.daemon=True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(f.client_url('control')) q.setsockopt_in(zmq.IDENTITY, b'control_in') q.bind_out(f.engine_url('control')) q.setsockopt_out(zmq.IDENTITY, b'control_out') q.connect_mon(monitor_url) q.daemon=True children.append(q) try: scheme = self.config.TaskScheduler.scheme_name except AttributeError: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure DEALER Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(f.client_url('task')) q.setsockopt_in(zmq.IDENTITY, b'task_in') q.bind_out(f.engine_url('task')) q.setsockopt_out(zmq.IDENTITY, b'task_out') q.connect_mon(monitor_url) q.daemon=True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler"%scheme) sargs = (f.client_url('task'), f.engine_url('task'), monitor_url, disambiguate_url(f.client_url('notification')), disambiguate_url(f.client_url('registration')), ) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url = self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon=True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs) def terminate_children(self): child_procs = [] for child in self.children: if isinstance(child, ProcessMonitoredQueue): child_procs.append(child.launcher) elif isinstance(child, Process): child_procs.append(child) if child_procs: self.log.critical("terminating children...") for child in child_procs: try: child.terminate() except OSError: # already dead pass def handle_signal(self, sig, frame): self.log.critical("Received signal %i, shutting down", sig) self.terminate_children() self.loop.stop() def init_signal(self): for sig in (SIGINT, SIGABRT, SIGTERM): signal(sig, self.handle_signal) def do_import_statements(self): statements = self.import_statements for s in statements: try: self.log.msg("Executing statement: '%s'" % s) exec s in globals(), locals() except: self.log.msg("Error running statement: %s" % s) def forward_logging(self): if self.log_url: self.log.info("Forwarding logging to %s"%self.log_url) context = zmq.Context.instance() lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = PUBHandler(lsock) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler) @catch_config_error def initialize(self, argv=None): super(IPControllerApp, self).initialize(argv) self.forward_logging() self.load_secondary_config() self.init_hub() self.init_schedulers() def start(self): # Start the subprocesses: self.factory.start() # children must be started before signals are setup, # otherwise signal-handling will fire multiple times for child in self.children: child.start() self.init_signal() self.write_pid_file(overwrite=True) try: self.factory.loop.start() except KeyboardInterrupt: self.log.critical("Interrupted, Exiting...\n") finally: self.cleanup_connection_files()
class NotebookManager(LoggingConfigurable): notebook_dir = Unicode(os.getcwd(), config=True, help=""" The directory to use for notebooks. """) def _notebook_dir_changed(self, name, old, new): """do a bit of validation of the notebook dir""" if os.path.exists(new) and not os.path.isdir(new): raise TraitError("notebook dir %r is not a directory" % new) if not os.path.exists(new): self.log.info("Creating notebook dir %s", new) try: os.mkdir(new) except: raise TraitError("Couldn't create notebook dir %r" % new) save_script = Bool( False, config=True, help="""Automatically create a Python script when saving the notebook. For easier use of import, %run and %load across notebooks, a <notebook-name>.py script will be created next to any <notebook-name>.ipynb on each save. This can also be set with the short `--script` flag. """) filename_ext = Unicode('.ipynb') allowed_formats = List(['json', 'py']) # Map notebook_ids to notebook names mapping = Dict() # Map notebook names to notebook_ids rev_mapping = Dict() def list_notebooks(self): """List all notebooks in the notebook dir. This returns a list of dicts of the form:: dict(notebook_id=notebook,name=name) """ names = glob.glob( os.path.join(self.notebook_dir, '*' + self.filename_ext)) names = [os.path.splitext(os.path.basename(name))[0] for name in names] data = [] for name in names: if name not in self.rev_mapping: notebook_id = self.new_notebook_id(name) else: notebook_id = self.rev_mapping[name] data.append(dict(notebook_id=notebook_id, name=name)) data = sorted(data, key=lambda item: item['name']) return data def new_notebook_id(self, name): """Generate a new notebook_id for a name and store its mappings.""" # TODO: the following will give stable urls for notebooks, but unless # the notebooks are immediately redirected to their new urls when their # filemname changes, nasty inconsistencies result. So for now it's # disabled and instead we use a random uuid4() call. But we leave the # logic here so that we can later reactivate it, whhen the necessary # url redirection code is written. #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL, # 'file://'+self.get_path_by_name(name).encode('utf-8'))) notebook_id = str(uuid.uuid4()) self.mapping[notebook_id] = name self.rev_mapping[name] = notebook_id return notebook_id def delete_notebook_id(self, notebook_id): """Delete a notebook's id only. This doesn't delete the actual notebook.""" name = self.mapping[notebook_id] del self.mapping[notebook_id] del self.rev_mapping[name] def notebook_exists(self, notebook_id): """Does a notebook exist?""" if notebook_id not in self.mapping: return False path = self.get_path_by_name(self.mapping[notebook_id]) return os.path.isfile(path) def find_path(self, notebook_id): """Return a full path to a notebook given its notebook_id.""" try: name = self.mapping[notebook_id] except KeyError: raise web.HTTPError(404, 'Notebook does not exist: %s' % notebook_id) return self.get_path_by_name(name) def get_path_by_name(self, name): """Return a full path to a notebook given its name.""" filename = name + self.filename_ext path = os.path.join(self.notebook_dir, filename) return path def get_notebook(self, notebook_id, format='json'): """Get the representation of a notebook in format by notebook_id.""" format = str(format) if format not in self.allowed_formats: raise web.HTTPError(415, 'Invalid notebook format: %s' % format) last_modified, nb = self.get_notebook_object(notebook_id) kwargs = {} if format == 'json': # don't split lines for sending over the wire, because it # should match the Python in-memory format. kwargs['split_lines'] = False data = current.writes(nb, format, **kwargs) name = nb.metadata.get('name', 'notebook') return last_modified, name, data def get_notebook_object(self, notebook_id): """Get the NotebookNode representation of a notebook by notebook_id.""" path = self.find_path(notebook_id) if not os.path.isfile(path): raise web.HTTPError(404, 'Notebook does not exist: %s' % notebook_id) info = os.stat(path) last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime) with open(path, 'r') as f: s = f.read() try: # v1 and v2 and json in the .ipynb files. nb = current.reads(s, 'json') except: raise web.HTTPError(500, 'Unreadable JSON notebook.') # Always use the filename as the notebook name. nb.metadata.name = os.path.splitext(os.path.basename(path))[0] return last_modified, nb def save_new_notebook(self, data, name=None, format='json'): """Save a new notebook and return its notebook_id. If a name is passed in, it overrides any values in the notebook data and the value in the data is updated to use that value. """ if format not in self.allowed_formats: raise web.HTTPError(415, 'Invalid notebook format: %s' % format) try: nb = current.reads(data.decode('utf-8'), format) except: raise web.HTTPError(400, 'Invalid JSON data') if name is None: try: name = nb.metadata.name except AttributeError: raise web.HTTPError(400, 'Missing notebook name') nb.metadata.name = name notebook_id = self.new_notebook_id(name) self.save_notebook_object(notebook_id, nb) return notebook_id def save_notebook(self, notebook_id, data, name=None, format='json'): """Save an existing notebook by notebook_id.""" if format not in self.allowed_formats: raise web.HTTPError(415, 'Invalid notebook format: %s' % format) try: nb = current.reads(data.decode('utf-8'), format) except: raise web.HTTPError(400, 'Invalid JSON data') if name is not None: nb.metadata.name = name self.save_notebook_object(notebook_id, nb) def save_notebook_object(self, notebook_id, nb): """Save an existing notebook object by notebook_id.""" if notebook_id not in self.mapping: raise web.HTTPError(404, 'Notebook does not exist: %s' % notebook_id) old_name = self.mapping[notebook_id] try: new_name = nb.metadata.name except AttributeError: raise web.HTTPError(400, 'Missing notebook name') path = self.get_path_by_name(new_name) try: with open(path, 'w') as f: current.write(nb, f, 'json') except Exception as e: raise web.HTTPError( 400, 'Unexpected error while saving notebook: %s' % e) # save .py script as well if self.save_script: pypath = os.path.splitext(path)[0] + '.py' try: with io.open(pypath, 'w', encoding='utf-8') as f: current.write(nb, f, 'py') except Exception as e: raise web.HTTPError( 400, 'Unexpected error while saving notebook as script: %s' % e) if old_name != new_name: old_path = self.get_path_by_name(old_name) if os.path.isfile(old_path): os.unlink(old_path) if self.save_script: old_pypath = os.path.splitext(old_path)[0] + '.py' if os.path.isfile(old_pypath): os.unlink(old_pypath) self.mapping[notebook_id] = new_name self.rev_mapping[new_name] = notebook_id del self.rev_mapping[old_name] def delete_notebook(self, notebook_id): """Delete notebook by notebook_id.""" path = self.find_path(notebook_id) if not os.path.isfile(path): raise web.HTTPError(404, 'Notebook does not exist: %s' % notebook_id) os.unlink(path) self.delete_notebook_id(notebook_id) def increment_filename(self, basename): """Return a non-used filename of the form basename<int>. This searches through the filenames (basename0, basename1, ...) until is find one that is not already being used. It is used to create Untitled and Copy names that are unique. """ i = 0 while True: name = '%s%i' % (basename, i) path = self.get_path_by_name(name) if not os.path.isfile(path): break else: i = i + 1 return path, name def new_notebook(self): """Create a new notebook and return its notebook_id.""" path, name = self.increment_filename('Untitled') notebook_id = self.new_notebook_id(name) metadata = current.new_metadata(name=name) nb = current.new_notebook(metadata=metadata) with open(path, 'w') as f: current.write(nb, f, 'json') return notebook_id def copy_notebook(self, notebook_id): """Copy an existing notebook and return its notebook_id.""" last_mod, nb = self.get_notebook_object(notebook_id) name = nb.metadata.name + '-Copy' path, name = self.increment_filename(name) nb.metadata.name = name notebook_id = self.new_notebook_id(name) self.save_notebook_object(notebook_id, nb) return notebook_id
class UnionTrait(HasTraits): value = Union([Type(), Bool()])
class HistoryAccessor(Configurable): """Access the history database without adding to it. This is intended for use by standalone history tools. IPython shells use HistoryManager, below, which is a subclass of this.""" # String holding the path to the history file hist_file = Unicode( config=True, help="""Path to file to use for SQLite history database. By default, IPython will put the history database in the IPython profile directory. If you would rather share one history among profiles, you can set this value in each, so that they are consistent. Due to an issue with fcntl, SQLite is known to misbehave on some NFS mounts. If you see IPython hanging, try setting this to something on a local disk, e.g:: ipython --HistoryManager.hist_file=/tmp/ipython_hist.sqlite """) enabled = Bool(True, config=True, help="""enable the SQLite history set enabled=False to disable the SQLite history, in which case there will be no stored history, no SQLite connection, and no background saving thread. This may be necessary in some threaded environments where IPython is embedded. """) connection_options = Dict( config=True, help="""Options for configuring the SQLite connection These options are passed as keyword args to sqlite3.connect when establishing database conenctions. """) # The SQLite database db = Any() def _db_changed(self, name, old, new): """validate the db, since it can be an Instance of two different types""" connection_types = (DummyDB, ) if sqlite3 is not None: connection_types = (DummyDB, sqlite3.Connection) if not isinstance(new, connection_types): msg = "%s.db must be sqlite3 Connection or DummyDB, not %r" % \ (self.__class__.__name__, new) raise TraitError(msg) def __init__(self, profile='default', hist_file=u'', **traits): """Create a new history accessor. Parameters ---------- profile : str The name of the profile from which to open history. hist_file : str Path to an SQLite history database stored by IPython. If specified, hist_file overrides profile. config : Config object. hist_file can also be set through this. """ # We need a pointer back to the shell for various tasks. super(HistoryAccessor, self).__init__(**traits) # defer setting hist_file from kwarg until after init, # otherwise the default kwarg value would clobber any value # set by config if hist_file: self.hist_file = hist_file if self.hist_file == u'': # No one has set the hist_file, yet. self.hist_file = self._get_hist_file_name(profile) if sqlite3 is None and self.enabled: warn( "IPython History requires SQLite, your history will not be saved" ) self.enabled = False self.init_db() def _get_hist_file_name(self, profile='default'): """Find the history file for the given profile name. This is overridden by the HistoryManager subclass, to use the shell's active profile. Parameters ---------- profile : str The name of a profile which has a history file. """ return os.path.join(locate_profile(profile), 'history.sqlite') @catch_corrupt_db def init_db(self): """Connect to the database, and create tables if necessary.""" if not self.enabled: self.db = DummyDB() return # use detect_types so that timestamps return datetime objects kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES) kwargs.update(self.connection_options) self.db = sqlite3.connect(self.hist_file, **kwargs) self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer primary key autoincrement, start timestamp, end timestamp, num_cmds integer, remark text)""") self.db.execute("""CREATE TABLE IF NOT EXISTS history (session integer, line integer, source text, source_raw text, PRIMARY KEY (session, line))""") # Output history is optional, but ensure the table's there so it can be # enabled later. self.db.execute("""CREATE TABLE IF NOT EXISTS output_history (session integer, line integer, output text, PRIMARY KEY (session, line))""") self.db.commit() def writeout_cache(self): """Overridden by HistoryManager to dump the cache before certain database lookups.""" pass ## ------------------------------- ## Methods for retrieving history: ## ------------------------------- def _run_sql(self, sql, params, raw=True, output=False): """Prepares and runs an SQL query for the history database. Parameters ---------- sql : str Any filtering expressions to go after SELECT ... FROM ... params : tuple Parameters passed to the SQL query (to replace "?") raw, output : bool See :meth:`get_range` Returns ------- Tuples as :meth:`get_range` """ toget = 'source_raw' if raw else 'source' sqlfrom = "history" if output: sqlfrom = "history LEFT JOIN output_history USING (session, line)" toget = "history.%s, output_history.output" % toget cur = self.db.execute("SELECT session, line, %s FROM %s " %\ (toget, sqlfrom) + sql, params) if output: # Regroup into 3-tuples, and parse JSON return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur) return cur @needs_sqlite @catch_corrupt_db def get_session_info(self, session=0): """get info about a session Parameters ---------- session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is previous session. Returns ------- (session_id [int], start [datetime], end [datetime], num_cmds [int], remark [unicode]) Sessions that are running or did not exit cleanly will have `end=None` and `num_cmds=None`. """ if session <= 0: session += self.session_number query = "SELECT * from sessions where session == ?" return self.db.execute(query, (session, )).fetchone() @catch_corrupt_db def get_tail(self, n=10, raw=True, output=False, include_latest=False): """Get the last n lines from the history database. Parameters ---------- n : int The number of lines to get raw, output : bool See :meth:`get_range` include_latest : bool If False (default), n+1 lines are fetched, and the latest one is discarded. This is intended to be used where the function is called by a user command, which it should not return. Returns ------- Tuples as :meth:`get_range` """ self.writeout_cache() if not include_latest: n += 1 cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?", (n, ), raw=raw, output=output) if not include_latest: return reversed(list(cur)[1:]) return reversed(list(cur)) @catch_corrupt_db def search(self, pattern="*", raw=True, search_raw=True, output=False, n=None, unique=False): """Search the database using unix glob-style matching (wildcards * and ?). Parameters ---------- pattern : str The wildcarded pattern to match when searching search_raw : bool If True, search the raw input, otherwise, the parsed input raw, output : bool See :meth:`get_range` n : None or int If an integer is given, it defines the limit of returned entries. unique : bool When it is true, return only unique entries. Returns ------- Tuples as :meth:`get_range` """ tosearch = "source_raw" if search_raw else "source" if output: tosearch = "history." + tosearch self.writeout_cache() sqlform = "WHERE %s GLOB ?" % tosearch params = (pattern, ) if unique: sqlform += ' GROUP BY {0}'.format(tosearch) if n is not None: sqlform += " ORDER BY session DESC, line DESC LIMIT ?" params += (n, ) elif unique: sqlform += " ORDER BY session, line" cur = self._run_sql(sqlform, params, raw=raw, output=output) if n is not None: return reversed(list(cur)) return cur @catch_corrupt_db def get_range(self, session, start=1, stop=None, raw=True, output=False): """Retrieve input by session. Parameters ---------- session : int Session number to retrieve. start : int First line to retrieve. stop : int End of line range (excluded from output itself). If None, retrieve to the end of the session. raw : bool If True, return untranslated input output : bool If True, attempt to include output. This will be 'real' Python objects for the current session, or text reprs from previous sessions if db_log_output was enabled at the time. Where no output is found, None is used. Returns ------- An iterator over the desired lines. Each line is a 3-tuple, either (session, line, input) if output is False, or (session, line, (input, output)) if output is True. """ if stop: lineclause = "line >= ? AND line < ?" params = (session, start, stop) else: lineclause = "line>=?" params = (session, start) return self._run_sql("WHERE session==? AND %s" % lineclause, params, raw=raw, output=output) def get_range_by_str(self, rangestr, raw=True, output=False): """Get lines of history from a string of ranges, as used by magic commands %hist, %save, %macro, etc. Parameters ---------- rangestr : str A string specifying ranges, e.g. "5 ~2/1-4". See :func:`magic_history` for full details. raw, output : bool As :meth:`get_range` Returns ------- Tuples as :meth:`get_range` """ for sess, s, e in extract_hist_ranges(rangestr): for line in self.get_range(sess, s, e, raw=raw, output=output): yield line