class BinaryView(DOMWidget): _view_module = Unicode('nbextensions/bitjet/bitjet', sync=True) _view_name = Unicode('BinaryView', sync=True) _model_module = Unicode('nbextensions/bitjet/bitjet', sync=True) _model_name = Unicode('BinaryModel', sync=True) datawidth = Int(2, sync=True) if ndarray: data = (Bytes(sync=True, to_json=b64encode_json) | Instance(ndarray, sync=True, to_json=b64encode_json)) else: data = Bytes(sync=True, to_json=b64encode_json) blockwidth = Int(4, sync=True) blockheight = Int(4, sync=True) height = Int(200, sync=True) width = Int(800, sync=True) bits_per_block = Enum([1, 8], default_value=1, sync=True)
class ImageWidget(DOMWidget): _view_name = Unicode('ImageView', sync=True) # Define the custom state properties to sync with the front-end format = Unicode('png', sync=True) width = CUnicode(sync=True) height = CUnicode(sync=True) _b64value = Unicode(sync=True) value = Bytes() def _value_changed(self, name, old, new): self._b64value = base64.b64encode(new)
class Image(DOMWidget): """Displays an image as a widget. The `value` of this widget accepts a byte string. The byte string is the raw image data that you want the browser to display. You can explicitly define the format of the byte string using the `format` trait (which defaults to "png").""" _view_name = Unicode('ImageView', sync=True) # Define the custom state properties to sync with the front-end format = Unicode('png', sync=True) width = CUnicode(sync=True) height = CUnicode(sync=True) _b64value = Unicode(sync=True) value = Bytes() def _value_changed(self, name, old, new): self._b64value = base64.b64encode(new)
class ImageButton(DOMWidget): disabled = Bool(False, help="Enable or disable user changes.", sync=True) _view_name = Unicode('ImageButtonView', sync=True) format = Unicode('png', sync=True) width = CUnicode(sync=True) height = CUnicode(sync=True) _b64value = Unicode(sync=True) value = Bytes() def _value_changed(self, name, old, new): self._b64value = base64.b64encode(new) def __init__(self, **kwargs): super(ImageButton, self).__init__(**kwargs) self._click_handlers = CallbackDispatcher() self.on_msg(self._handle_button_msg) def on_click(self, callback, remove=False): self._click_handlers.register_callback(callback, remove=remove) def _handle_button_msg(self, _, content): if content.get('event', '') == 'click': self._click_handlers(self, content)
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples aliases = aliases flags = flags classes = [ KernelManager, ProfileDir, Session, MappingKernelManager, ContentsManager, FileContentsManager, NotebookNotary, KernelSpecManager, ] flags = Dict(flags) aliases = Dict(aliases) subcommands = dict( list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), ) ipython_kernel_argv = List(Unicode) _log_formatter_cls = LogFormatter def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%H:%M:%S" def _log_format_default(self): """override default log format to include time""" return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('', config=True) # Network related information allow_origin = Unicode('', config=True, help="""Set the Access-Control-Allow-Origin header Use '*' to allow any origin to access your server. Takes precedence over allow_origin_pat. """ ) allow_origin_pat = Unicode('', config=True, help="""Use a regular expression for the Access-Control-Allow-Origin header Requests from an origin matching the expression will get replies with: Access-Control-Allow-Origin: origin where `origin` is the origin of the request. Ignored if allow_origin is set. """ ) allow_credentials = Bool(False, config=True, help="Set the Access-Control-Allow-Credentials: true header" ) default_url = Unicode('/tree', config=True, help="The default URL to redirect to from `/`" ) ip = Unicode('localhost', config=True, help="The IP address the notebook server will listen on." ) def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on." ) port_retries = Integer(50, config=True, help="The number of additional ports to try if the specified port is not available." ) certfile = Unicode(u'', config=True, help="""The full path to an SSL/TLS certificate file.""" ) keyfile = Unicode(u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""" ) cookie_secret_file = Unicode(config=True, help="""The file where the cookie secret is stored.""" ) def _cookie_secret_file_default(self): if self.profile_dir is None: return '' return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret') cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """ ) def _cookie_secret_default(self): if os.path.exists(self.cookie_secret_file): with io.open(self.cookie_secret_file, 'rb') as f: return f.read() else: secret = base64.encodestring(os.urandom(1024)) self._write_cookie_secret_file(secret) return secret def _write_cookie_secret_file(self, secret): """write my secret to my secret_file""" self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file) with io.open(self.cookie_secret_file, 'wb') as f: f.write(secret) try: os.chmod(self.cookie_secret_file, 0o600) except OSError: self.log.warn( "Could not set permissions on %s", self.cookie_secret_file ) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """ ) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict(config=True, help="DEPRECATED, use tornado_settings" ) def _webapp_settings_changed(self, name, old, new): self.log.warn("\n webapp_settings is deprecated, use tornado_settings.\n") self.tornado_settings = new tornado_settings = Dict(config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") ssl_options = Dict(config=True, help="""Supply SSL options for the tornado HTTPServer. See the tornado docs for details.""") jinja_environment_options = Dict(config=True, help="Supply extra arguments that will be passed to Jinja environment.") enable_mathjax = Bool(True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """ ) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_url_changed(self, name, old, new): if not new.startswith('/'): self.base_url = '/'+new elif not new.endswith('/'): self.base_url = new+'/' base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""") def _base_project_url_changed(self, name, old, new): self.log.warn("base_project_url is deprecated, use base_url") self.base_url = new extra_static_paths = List(Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""" ) def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] extra_template_paths = List(Unicode, config=True, help="""Extra paths to search for serving jinja templates. Can be used to override templates from IPython.html.templates.""" ) def _extra_template_paths_default(self): return [] @property def template_file_path(self): """return extra paths + the default locations""" return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST extra_nbextensions_path = List(Unicode, config=True, help="""extra paths to look for Javascript notebook extensions""" ) @property def nbextensions_path(self): """The path to look for Javascript notebook extensions""" return self.extra_nbextensions_path + [os.path.join(get_ipython_dir(), 'nbextensions')] + SYSTEM_NBEXTENSIONS_DIRS websocket_url = Unicode("", config=True, help="""The base URL for websockets, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """ ) mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""" ) def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.tornado_settings.get("static_url_prefix", url_path_join(self.base_url, "static") ) # try local mathjax, either in nbextensions/mathjax or static/mathjax for (url_prefix, search_path) in [ (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path), (static_url_prefix, self.static_file_path), ]: self.log.debug("searching for local mathjax in %s", search_path) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path) except IOError: continue else: url = url_path_join(url_prefix, u"mathjax/MathJax.js") self.log.info("Serving local MathJax from %s at %s", mathjax, url) return url # no local mathjax, serve from CDN url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) contents_manager_class = Type( default_value=FileContentsManager, klass=ContentsManager, config=True, help='The notebook manager class to use.' ) kernel_manager_class = Type( default_value=MappingKernelManager, config=True, help='The kernel manager class to use.' ) session_manager_class = Type( default_value=SessionManager, config=True, help='The session manager class to use.' ) cluster_manager_class = Type( default_value=ClusterManager, config=True, help='The cluster manager class to use.' ) config_manager_class = Type( default_value=ConfigManager, config = True, help='The config manager class to use' ) kernel_spec_manager = Instance(KernelSpecManager) kernel_spec_manager_class = Type( default_value=KernelSpecManager, config=True, help=""" The kernel spec manager class to use. Should be a subclass of `IPython.kernel.kernelspec.KernelSpecManager`. The Api of KernelSpecManager is provisional and might change without warning between this version of IPython and the next stable one. """ ) login_handler_class = Type( default_value=LoginHandler, klass=web.RequestHandler, config=True, help='The login handler class to use.', ) logout_handler_class = Type( default_value=LogoutHandler, klass=web.RequestHandler, config=True, help='The logout handler class to use.', ) trust_xheaders = Bool(False, config=True, help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL") ) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json"%os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) pylab = Unicode('disabled', config=True, help=""" DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. """ ) def _pylab_changed(self, name, old, new): """when --pylab is specified, display a warning and exit""" if new != 'warn': backend = ' %s' % new else: backend = '' self.log.error("Support for specifying --pylab on the command line has been removed.") self.log.error( "Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself.".format(backend) ) self.exit(1) notebook_dir = Unicode(config=True, help="The directory to use for notebooks and kernels." ) def _notebook_dir_default(self): if self.file_to_run: return os.path.dirname(os.path.abspath(self.file_to_run)) else: return py3compat.getcwd() def _notebook_dir_changed(self, name, old, new): """Do a bit of validation of the notebook dir.""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. self.notebook_dir = os.path.abspath(new) return if not os.path.isdir(new): raise TraitError("No such notebook dir: %r" % new) # setting App.notebook_dir implies setting notebook and kernel dirs as well self.config.FileContentsManager.root_dir = new self.config.MappingKernelManager.root_dir = new server_extensions = List(Unicode(), config=True, help=("Python modules to load as notebook server extensions. " "This is an experimental API, and may change in future releases.") ) def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: arg0 = self.extra_args[0] f = os.path.abspath(arg0) self.argv.remove(arg0) if not os.path.exists(f): self.log.critical("No such file or directory: %s", f) self.exit(1) # Use config here, to ensure that it takes higher priority than # anything that comes from the profile. c = Config() if os.path.isdir(f): c.NotebookApp.notebook_dir = f elif os.path.isfile(f): c.NotebookApp.file_to_run = f self.update_config(c) def init_kernel_argv(self): """add the profile-dir to arguments to be passed to IPython kernels""" # FIXME: remove special treatment of IPython kernels # Kernel should get *absolute* path to profile directory self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location] def init_configurables(self): self.kernel_spec_manager = self.kernel_spec_manager_class( parent=self, ipython_dir=self.ipython_dir, ) self.kernel_manager = self.kernel_manager_class( parent=self, log=self.log, ipython_kernel_argv=self.ipython_kernel_argv, connection_dir=self.profile_dir.security_dir, ) self.contents_manager = self.contents_manager_class( parent=self, log=self.log, ) self.session_manager = self.session_manager_class( parent=self, log=self.log, kernel_manager=self.kernel_manager, contents_manager=self.contents_manager, ) self.cluster_manager = self.cluster_manager_class( parent=self, log=self.log, ) self.config_manager = self.config_manager_class( parent=self, log=self.log, profile_dir=self.profile_dir.location, ) def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False for log in app_log, access_log, gen_log: # consistent log output name (NotebookApp instead of tornado.access, etc.) log.name = self.log.name # hook up tornado 3's loggers to our app handlers logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.tornado_settings['allow_origin'] = self.allow_origin if self.allow_origin_pat: self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat) self.tornado_settings['allow_credentials'] = self.allow_credentials # ensure default_url starts with base_url if not self.default_url.startswith(self.base_url): self.default_url = url_path_join(self.base_url, self.default_url) self.web_app = NotebookWebApplication( self, self.kernel_manager, self.contents_manager, self.cluster_manager, self.session_manager, self.kernel_spec_manager, self.config_manager, self.log, self.base_url, self.default_url, self.tornado_settings, self.jinja_environment_options ) ssl_options = self.ssl_options if self.certfile: ssl_options['certfile'] = self.certfile if self.keyfile: ssl_options['keyfile'] = self.keyfile if not ssl_options: # None indicates no SSL config ssl_options = None self.login_handler_class.validate_security(self, ssl_options=ssl_options) self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) success = None for port in random_ports(self.port, self.port_retries+1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info('The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical('ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else 'localhost' return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) def init_terminals(self): try: from .terminal import initialize initialize(self.web_app) self.web_app.settings['terminals_available'] = True except ImportError as e: log = self.log.debug if sys.platform == 'win32' else self.log.warn log("Terminals not available (error was %s)", e) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r,w,x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y') and 'n' not in line.lower(): self.log.critical("Shutdown confirmed") ioloop.IOLoop.current().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.current().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.current().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn("components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn("components submodule unclean, you may see 404s on static/components") self.log.warn("run `setup.py submodule` or `git submodule update` to update") def init_server_extensions(self): """Load any extensions specified by config. Import the module, then call the load_jupyter_server_extension function, if one exists. The extension API is experimental, and may change in future releases. """ for modulename in self.server_extensions: try: mod = importlib.import_module(modulename) func = getattr(mod, 'load_jupyter_server_extension', None) if func is not None: func(self) except Exception: self.log.warn("Error loading server extension %s", modulename, exc_info=True) @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_terminals() self.init_signal() self.init_server_extensions() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.contents_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return {'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), 'base_url': self.base_url, 'notebook_dir': os.path.abspath(self.notebook_dir), 'pid': os.getpid() } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).") self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: if not os.path.exists(self.file_to_run): self.log.critical("%s does not exist" % self.file_to_run) self.exit(1) relpath = os.path.relpath(self.file_to_run, self.notebook_dir) uri = url_path_join('notebooks', *relpath.split(os.sep)) else: uri = 'tree' if browser: b = lambda : browser.open(url_path_join(self.connection_url, uri), new=2) threading.Thread(target=b).start() self.io_loop = ioloop.IOLoop.current() if sys.platform.startswith('win'): # add no-op to wake every 5s # to handle signals that may be ignored by the inner loop pc = ioloop.PeriodicCallback(lambda : None, 5000) pc.start() try: self.io_loop.start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file() def stop(self): def _stop(): self.http_server.stop() self.io_loop.stop() self.io_loop.add_callback(_stop)
class Comm(LoggingConfigurable): shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') def _shell_default(self): return get_ipython() iopub_socket = Any() def _iopub_socket_default(self): return self.shell.kernel.iopub_socket session = Instance('IPython.kernel.zmq.session.Session') def _session_default(self): if self.shell is None or not hasattr(self.shell, 'kernel'): return return self.shell.kernel.session target_name = Unicode('comm') topic = Bytes() def _topic_default(self): return ('comm-%s' % self.comm_id).encode('ascii') _open_data = Dict(help="data dict, if any, to be included in comm_open") _close_data = Dict(help="data dict, if any, to be included in comm_close") _msg_callback = Any() _close_callback = Any() _closed = Bool(False) comm_id = Unicode() def _comm_id_default(self): return uuid.uuid4().hex primary = Bool(True, help="Am I the primary or secondary Comm?") def __init__(self, target_name='', data=None, **kwargs): if target_name: kwargs['target_name'] = target_name super(Comm, self).__init__(**kwargs) if self.primary: # I am primary, open my peer. self.open(data) def _publish_msg(self, msg_type, data=None, metadata=None, **keys): """Helper for sending a comm message on IOPub""" if self.session is not None: data = {} if data is None else data metadata = {} if metadata is None else metadata content = json_clean(dict(data=data, comm_id=self.comm_id, **keys)) self.session.send(self.iopub_socket, msg_type, content, metadata=json_clean(metadata), parent=self.shell.get_parent(), ident=self.topic, ) def __del__(self): """trigger close on gc""" self.close() # publishing messages def open(self, data=None, metadata=None): """Open the frontend-side version of this comm""" if data is None: data = self._open_data self._closed = False ip = get_ipython() if hasattr(ip, 'comm_manager'): ip.comm_manager.register_comm(self) self._publish_msg('comm_open', data, metadata, target_name=self.target_name) def close(self, data=None, metadata=None): """Close the frontend-side version of this comm""" if self._closed: # only close once return if data is None: data = self._close_data self._publish_msg('comm_close', data, metadata) ip = get_ipython() if hasattr(ip, 'comm_manager'): ip.comm_manager.unregister_comm(self) self._closed = True def send(self, data=None, metadata=None): """Send a message to the frontend-side version of this comm""" self._publish_msg('comm_msg', data, metadata) # registering callbacks def on_close(self, callback): """Register a callback for comm_close Will be called with the `data` of the close message. Call `on_close(None)` to disable an existing callback. """ self._close_callback = callback def on_msg(self, callback): """Register a callback for comm_msg Will be called with the `data` of any comm_msg messages. Call `on_msg(None)` to disable an existing callback. """ self._msg_callback = callback # handling of incoming messages def handle_close(self, msg): """Handle a comm_close message""" self.log.debug("handle_close[%s](%s)", self.comm_id, msg) if self._close_callback: self._close_callback(msg) def handle_msg(self, msg): """Handle a comm_msg message""" self.log.debug("handle_msg[%s](%s)", self.comm_id, msg) if self._msg_callback: self.shell.events.trigger('pre_execute') self._msg_callback(msg) self.shell.events.trigger('post_execute')
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples aliases = aliases flags = flags classes = [ KernelManager, ProfileDir, Session, MappingKernelManager, NotebookManager, FileNotebookManager, NotebookNotary, ] flags = Dict(flags) aliases = Dict(aliases) subcommands = dict(list=(NbserverListApp, NbserverListApp.description.splitlines()[0]), ) kernel_argv = List(Unicode) _log_formatter_cls = LogFormatter def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%H:%M:%S" def _log_format_default(self): """override default log format to include time""" return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('', config=True) def _file_to_run_changed(self, name, old, new): path, base = os.path.split(new) if path: self.file_to_run = base self.notebook_dir = path # Network related information. ip = Unicode('localhost', config=True, help="The IP address the notebook server will listen on.") def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on.") port_retries = Integer( 50, config=True, help= "The number of additional ports to try if the specified port is not available." ) certfile = Unicode( u'', config=True, help="""The full path to an SSL/TLS certificate file.""") keyfile = Unicode( u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""") cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """) def _cookie_secret_default(self): return os.urandom(1024) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict( config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") jinja_environment_options = Dict( config=True, help="Supply extra arguments that will be passed to Jinja environment." ) enable_mathjax = Bool( True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_url_changed(self, name, old, new): if not new.startswith('/'): self.base_url = '/' + new elif not new.endswith('/'): self.base_url = new + '/' base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""") def _base_project_url_changed(self, name, old, new): self.log.warn("base_project_url is deprecated, use base_url") self.base_url = new extra_static_paths = List( Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""") def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] nbextensions_path = List( Unicode, config=True, help= """paths for Javascript extensions. By default, this is just IPYTHONDIR/nbextensions""" ) def _nbextensions_path_default(self): return [os.path.join(get_ipython_dir(), 'nbextensions')] mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""") def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.webapp_settings.get( "static_url_prefix", url_path_join(self.base_url, "static")) # try local mathjax, either in nbextensions/mathjax or static/mathjax for (url_prefix, search_path) in [ (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path), (static_url_prefix, self.static_file_path), ]: self.log.debug("searching for local mathjax in %s", search_path) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path) except IOError: continue else: url = url_path_join(url_prefix, u"mathjax/MathJax.js") self.log.info("Serving local MathJax from %s at %s", mathjax, url) return url # no local mathjax, serve from CDN if self.certfile: # HTTPS: load from Rackspace CDN, because SSL certificate requires it host = u"https://c328740.ssl.cf1.rackcdn.com" else: host = u"http://cdn.mathjax.org" url = host + u"/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) notebook_manager_class = DottedObjectName( 'IPython.html.services.notebooks.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') trust_xheaders = Bool( False, config=True, help= ("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL" )) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json" % os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) notebook_dir = Unicode( py3compat.getcwd(), config=True, help="The directory to use for notebooks and kernels.") pylab = Unicode('disabled', config=True, help=""" DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib. """) def _pylab_changed(self, name, old, new): """when --pylab is specified, display a warning and exit""" if new != 'warn': backend = ' %s' % new else: backend = '' self.log.error( "Support for specifying --pylab on the command line has been removed." ) self.log.error( "Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself." .format(backend)) self.exit(1) def _notebook_dir_changed(self, name, old, new): """Do a bit of validation of the notebook dir.""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. self.notebook_dir = os.path.abspath(new) return if not os.path.isdir(new): raise TraitError("No such notebook dir: %r" % new) # setting App.notebook_dir implies setting notebook and kernel dirs as well self.config.FileNotebookManager.notebook_dir = new self.config.MappingKernelManager.root_dir = new def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if self.extra_args: arg0 = self.extra_args[0] f = os.path.abspath(arg0) self.argv.remove(arg0) if not os.path.exists(f): self.log.critical("No such file or directory: %s", f) self.exit(1) # Use config here, to ensure that it takes higher priority than # anything that comes from the profile. c = Config() if os.path.isdir(f): c.NotebookApp.notebook_dir = f elif os.path.isfile(f): c.NotebookApp.file_to_run = f self.update_config(c) def init_kernel_argv(self): """construct the kernel arguments""" self.kernel_argv = [] # Kernel should inherit default config file from frontend self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) # Kernel should get *absolute* path to profile directory self.kernel_argv.extend(["--profile-dir", self.profile_dir.location]) def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_manager = MappingKernelManager( parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir=self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(parent=self, log=self.log) self.session_manager = SessionManager(parent=self, log=self.log) self.cluster_manager = ClusterManager(parent=self, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False # hook up tornado 3's loggers to our app handlers logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication( self, self.kernel_manager, self.notebook_manager, self.cluster_manager, self.session_manager, self.log, self.base_url, self.webapp_settings, self.jinja_environment_options) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical( warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries + 1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info( 'The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical( 'ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else 'localhost' return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 time.sleep(0.1) info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r, w, x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y') and 'n' not in line.lower(): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn( "components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn( "components submodule unclean, you may see 404s on static/components" ) self.log.warn( "run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): super(NotebookApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.notebook_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return { 'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), 'base_url': self.base_url, 'notebook_dir': os.path.abspath(self.notebook_dir), } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) self.write_server_info_file() if self.open_browser or self.file_to_run: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: fullpath = os.path.join(self.notebook_dir, self.file_to_run) if not os.path.exists(fullpath): self.log.critical("%s does not exist" % fullpath) self.exit(1) uri = url_path_join('notebooks', self.file_to_run) else: uri = 'tree' if browser: b = lambda: browser.open( url_path_join(self.connection_url, uri), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file()
class JupyterHub(Application): """An Application for starting a Multi-User Jupyter Notebook server.""" name = 'jupyterhub' description = """Start a multi-user Jupyter Notebook server Spawns a configurable-http-proxy and multi-user Hub, which authenticates users and spawns single-user Notebook servers on behalf of users. """ examples = """ generate default config file: jupyterhub --generate-config -f /etc/jupyterhub/jupyterhub.py spawn the server on 10.0.1.2:443 with https: jupyterhub --ip 10.0.1.2 --port 443 --ssl-key my_ssl.key --ssl-cert my_ssl.cert """ aliases = Dict(aliases) flags = Dict(flags) subcommands = {'token': (NewToken, "Generate an API token for a user")} classes = List([ Spawner, LocalProcessSpawner, Authenticator, PAMAuthenticator, ]) config_file = Unicode( 'jupyterhub_config.py', config=True, help="The config file to load", ) generate_config = Bool( False, config=True, help="Generate default config file", ) answer_yes = Bool( False, config=True, help="Answer yes to any questions (e.g. confirm overwrite)") pid_file = Unicode('', config=True, help="""File to write PID Useful for daemonizing jupyterhub. """) last_activity_interval = Integer( 300, config=True, help= "Interval (in seconds) at which to update last-activity timestamps.") proxy_check_interval = Integer( 30, config=True, help="Interval (in seconds) at which to check if the proxy is running." ) data_files_path = Unicode( DATA_FILES_PATH, config=True, help= "The location of jupyterhub data files (e.g. /usr/local/share/jupyter/hub)" ) ssl_key = Unicode( '', config=True, help="""Path to SSL key file for the public facing interface of the proxy Use with ssl_cert """) ssl_cert = Unicode( '', config=True, help= """Path to SSL certificate file for the public facing interface of the proxy Use with ssl_key """) ip = Unicode('', config=True, help="The public facing ip of the proxy") port = Integer(8000, config=True, help="The public facing port of the proxy") base_url = URLPrefix('/', config=True, help="The base URL of the entire application") jinja_environment_options = Dict( config=True, help="Supply extra arguments that will be passed to Jinja environment." ) proxy_cmd = Unicode('configurable-http-proxy', config=True, help="""The command to start the http proxy. Only override if configurable-http-proxy is not on your PATH """) debug_proxy = Bool(False, config=True, help="show debug output in configurable-http-proxy") proxy_auth_token = Unicode(config=True, help="""The Proxy Auth token. Loaded from the CONFIGPROXY_AUTH_TOKEN env variable by default. """) def _proxy_auth_token_default(self): token = os.environ.get('CONFIGPROXY_AUTH_TOKEN', None) if not token: self.log.warn('\n'.join([ "", "Generating CONFIGPROXY_AUTH_TOKEN. Restarting the Hub will require restarting the proxy.", "Set CONFIGPROXY_AUTH_TOKEN env or JupyterHub.proxy_auth_token config to avoid this message.", "", ])) token = orm.new_token() return token proxy_api_ip = Unicode('localhost', config=True, help="The ip for the proxy API handlers") proxy_api_port = Integer(config=True, help="The port for the proxy API handlers") def _proxy_api_port_default(self): return self.port + 1 hub_port = Integer(8081, config=True, help="The port for this process") hub_ip = Unicode('localhost', config=True, help="The ip for this process") hub_prefix = URLPrefix( '/hub/', config=True, help="The prefix for the hub server. Must not be '/'") def _hub_prefix_default(self): return url_path_join(self.base_url, '/hub/') def _hub_prefix_changed(self, name, old, new): if new == '/': raise TraitError("'/' is not a valid hub prefix") if not new.startswith(self.base_url): self.hub_prefix = url_path_join(self.base_url, new) cookie_secret = Bytes(config=True, env='JPY_COOKIE_SECRET', help="""The cookie secret to use to encrypt cookies. Loaded from the JPY_COOKIE_SECRET env variable by default. """) cookie_secret_file = Unicode( 'jupyterhub_cookie_secret', config=True, help="""File in which to store the cookie secret.""") authenticator_class = Type(PAMAuthenticator, Authenticator, config=True, help="""Class for authenticating users. This should be a class with the following form: - constructor takes one kwarg: `config`, the IPython config object. - is a tornado.gen.coroutine - returns username on success, None on failure - takes two arguments: (handler, data), where `handler` is the calling web.RequestHandler, and `data` is the POST form data from the login page. """) authenticator = Instance(Authenticator) def _authenticator_default(self): return self.authenticator_class(parent=self, db=self.db) # class for spawning single-user servers spawner_class = Type( LocalProcessSpawner, Spawner, config=True, help="""The class to use for spawning single-user servers. Should be a subclass of Spawner. """) db_url = Unicode( 'sqlite:///jupyterhub.sqlite', config=True, help="url for the database. e.g. `sqlite:///jupyterhub.sqlite`") def _db_url_changed(self, name, old, new): if '://' not in new: # assume sqlite, if given as a plain filename self.db_url = 'sqlite:///%s' % new db_kwargs = Dict( config=True, help="""Include any kwargs to pass to the database connection. See sqlalchemy.create_engine for details. """) reset_db = Bool(False, config=True, help="Purge and reset the database.") debug_db = Bool( False, config=True, help="log all database transactions. This has A LOT of output") db = Any() session_factory = Any() admin_access = Bool( False, config=True, help="""Grant admin users permission to access single-user servers. Users should be properly informed if this is enabled. """) admin_users = Set(config=True, help="""set of usernames of admin users If unspecified, only the user that launches the server will be admin. """) tornado_settings = Dict(config=True) cleanup_servers = Bool( True, config=True, help="""Whether to shutdown single-user servers when the Hub shuts down. Disable if you want to be able to teardown the Hub while leaving the single-user servers running. If both this and cleanup_proxy are False, sending SIGINT to the Hub will only shutdown the Hub, leaving everything else running. The Hub should be able to resume from database state. """) cleanup_proxy = Bool( True, config=True, help="""Whether to shutdown the proxy when the Hub shuts down. Disable if you want to be able to teardown the Hub while leaving the proxy running. Only valid if the proxy was starting by the Hub process. If both this and cleanup_servers are False, sending SIGINT to the Hub will only shutdown the Hub, leaving everything else running. The Hub should be able to resume from database state. """) handlers = List() _log_formatter_cls = CoroutineLogFormatter http_server = None proxy_process = None io_loop = None def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%Y-%m-%d %H:%M:%S" def _log_format_default(self): """override default log format to include time""" return "%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s %(module)s:%(lineno)d]%(end_color)s %(message)s" extra_log_file = Unicode("", config=True, help="Set a logging.FileHandler on this file.") extra_log_handlers = List( Instance(logging.Handler), config=True, help="Extra log handlers to set on JupyterHub logger", ) def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False if self.extra_log_file: self.extra_log_handlers.append( logging.FileHandler(self.extra_log_file)) _formatter = self._log_formatter_cls( fmt=self.log_format, datefmt=self.log_datefmt, ) for handler in self.extra_log_handlers: if handler.formatter is None: handler.setFormatter(_formatter) self.log.addHandler(handler) # hook up tornado 3's loggers to our app handlers for log in (app_log, access_log, gen_log): # ensure all log statements identify the application they come from log.name = self.log.name logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_ports(self): if self.hub_port == self.port: raise TraitError( "The hub and proxy cannot both listen on port %i" % self.port) if self.hub_port == self.proxy_api_port: raise TraitError( "The hub and proxy API cannot both listen on port %i" % self.hub_port) if self.proxy_api_port == self.port: raise TraitError( "The proxy's public and API ports cannot both be %i" % self.port) @staticmethod def add_url_prefix(prefix, handlers): """add a url prefix to handlers""" for i, tup in enumerate(handlers): lis = list(tup) lis[0] = url_path_join(prefix, tup[0]) handlers[i] = tuple(lis) return handlers def init_handlers(self): h = [] h.extend(handlers.default_handlers) h.extend(apihandlers.default_handlers) # load handlers from the authenticator h.extend(self.authenticator.get_handlers(self)) self.handlers = self.add_url_prefix(self.hub_prefix, h) # some extra handlers, outside hub_prefix self.handlers.extend([ (r"%s" % self.hub_prefix.rstrip('/'), web.RedirectHandler, { "url": self.hub_prefix, "permanent": False, }), (r"(?!%s).*" % self.hub_prefix, handlers.PrefixRedirectHandler), (r'(.*)', handlers.Template404), ]) def _check_db_path(self, path): """More informative log messages for failed filesystem access""" path = os.path.abspath(path) parent, fname = os.path.split(path) user = getuser() if not os.path.isdir(parent): self.log.error("Directory %s does not exist", parent) if os.path.exists(parent) and not os.access(parent, os.W_OK): self.log.error("%s cannot create files in %s", user, parent) if os.path.exists(path) and not os.access(path, os.W_OK): self.log.error("%s cannot edit %s", user, path) def init_secrets(self): trait_name = 'cookie_secret' trait = self.traits()[trait_name] env_name = trait.get_metadata('env') secret_file = os.path.abspath( os.path.expanduser(self.cookie_secret_file)) secret = self.cookie_secret secret_from = 'config' # load priority: 1. config, 2. env, 3. file if not secret and os.environ.get(env_name): secret_from = 'env' self.log.info("Loading %s from env[%s]", trait_name, env_name) secret = binascii.a2b_hex(os.environ[env_name]) if not secret and os.path.exists(secret_file): secret_from = 'file' perm = os.stat(secret_file).st_mode if perm & 0o077: self.log.error("Bad permissions on %s", secret_file) else: self.log.info("Loading %s from %s", trait_name, secret_file) with open(secret_file) as f: b64_secret = f.read() try: secret = binascii.a2b_base64(b64_secret) except Exception as e: self.log.error("%s does not contain b64 key: %s", secret_file, e) if not secret: secret_from = 'new' self.log.debug("Generating new %s", trait_name) secret = os.urandom(SECRET_BYTES) if secret_file and secret_from == 'new': # if we generated a new secret, store it in the secret_file self.log.info("Writing %s to %s", trait_name, secret_file) b64_secret = binascii.b2a_base64(secret).decode('ascii') with open(secret_file, 'w') as f: f.write(b64_secret) try: os.chmod(secret_file, 0o600) except OSError: self.log.warn("Failed to set permissions on %s", secret_file) # store the loaded trait value self.cookie_secret = secret def init_db(self): """Create the database connection""" self.log.debug("Connecting to db: %s", self.db_url) try: self.session_factory = orm.new_session_factory(self.db_url, reset=self.reset_db, echo=self.debug_db, **self.db_kwargs) self.db = scoped_session(self.session_factory)() except OperationalError as e: self.log.error("Failed to connect to db: %s", self.db_url) self.log.debug("Database error was:", exc_info=True) if self.db_url.startswith('sqlite:///'): self._check_db_path(self.db_url.split(':///', 1)[1]) self.exit(1) def init_hub(self): """Load the Hub config into the database""" self.hub = self.db.query(orm.Hub).first() if self.hub is None: self.hub = orm.Hub(server=orm.Server( ip=self.hub_ip, port=self.hub_port, base_url=self.hub_prefix, cookie_name='jupyter-hub-token', )) self.db.add(self.hub) else: server = self.hub.server server.ip = self.hub_ip server.port = self.hub_port server.base_url = self.hub_prefix self.db.commit() @gen.coroutine def init_users(self): """Load users into and from the database""" db = self.db if not self.admin_users: # add current user as admin if there aren't any others admins = db.query(orm.User).filter(orm.User.admin == True) if admins.first() is None: self.admin_users.add(getuser()) new_users = [] for name in self.admin_users: # ensure anyone specified as admin in config is admin in db user = orm.User.find(db, name) if user is None: user = orm.User(name=name, admin=True) new_users.append(user) db.add(user) else: user.admin = True # the admin_users config variable will never be used after this point. # only the database values will be referenced. whitelist = self.authenticator.whitelist if not whitelist: self.log.info( "Not using whitelist. Any authenticated user will be allowed.") # add whitelisted users to the db for name in whitelist: user = orm.User.find(db, name) if user is None: user = orm.User(name=name) new_users.append(user) db.add(user) if whitelist: # fill the whitelist with any users loaded from the db, # so we are consistent in both directions. # This lets whitelist be used to set up initial list, # but changes to the whitelist can occur in the database, # and persist across sessions. for user in db.query(orm.User): whitelist.add(user.name) # The whitelist set and the users in the db are now the same. # From this point on, any user changes should be done simultaneously # to the whitelist set and user db, unless the whitelist is empty (all users allowed). db.commit() for user in new_users: yield gen.maybe_future(self.authenticator.add_user(user)) db.commit() user_summaries = [''] def _user_summary(user): parts = ['{0: >8}'.format(user.name)] if user.admin: parts.append('admin') if user.server: parts.append('running at %s' % user.server) return ' '.join(parts) @gen.coroutine def user_stopped(user): status = yield user.spawner.poll() self.log.warn( "User %s server stopped with exit code: %s", user.name, status, ) yield self.proxy.delete_user(user) yield user.stop() for user in db.query(orm.User): if not user.state: # without spawner state, server isn't valid user.server = None user_summaries.append(_user_summary(user)) continue self.log.debug("Loading state for %s from db", user.name) user.spawner = spawner = self.spawner_class( user=user, hub=self.hub, config=self.config, db=self.db, ) status = yield spawner.poll() if status is None: self.log.info("%s still running", user.name) spawner.add_poll_callback(user_stopped, user) spawner.start_polling() else: # user not running. This is expected if server is None, # but indicates the user's server died while the Hub wasn't running # if user.server is defined. log = self.log.warn if user.server else self.log.debug log("%s not running.", user.name) user.server = None user_summaries.append(_user_summary(user)) self.log.debug("Loaded users: %s", '\n'.join(user_summaries)) db.commit() def init_proxy(self): """Load the Proxy config into the database""" self.proxy = self.db.query(orm.Proxy).first() if self.proxy is None: self.proxy = orm.Proxy( public_server=orm.Server(), api_server=orm.Server(), ) self.db.add(self.proxy) self.db.commit() self.proxy.auth_token = self.proxy_auth_token # not persisted self.proxy.log = self.log self.proxy.public_server.ip = self.ip self.proxy.public_server.port = self.port self.proxy.api_server.ip = self.proxy_api_ip self.proxy.api_server.port = self.proxy_api_port self.proxy.api_server.base_url = '/api/routes/' self.db.commit() @gen.coroutine def start_proxy(self): """Actually start the configurable-http-proxy""" # check for proxy if self.proxy.public_server.is_up() or self.proxy.api_server.is_up(): # check for *authenticated* access to the proxy (auth token can change) try: yield self.proxy.get_routes() except (HTTPError, OSError, socket.error) as e: if isinstance(e, HTTPError) and e.code == 403: msg = "Did CONFIGPROXY_AUTH_TOKEN change?" else: msg = "Is something else using %s?" % self.proxy.public_server.url self.log.error( "Proxy appears to be running at %s, but I can't access it (%s)\n%s", self.proxy.public_server.url, e, msg) self.exit(1) return else: self.log.info("Proxy already running at: %s", self.proxy.public_server.url) self.proxy_process = None return env = os.environ.copy() env['CONFIGPROXY_AUTH_TOKEN'] = self.proxy.auth_token cmd = [ self.proxy_cmd, '--ip', self.proxy.public_server.ip, '--port', str(self.proxy.public_server.port), '--api-ip', self.proxy.api_server.ip, '--api-port', str(self.proxy.api_server.port), '--default-target', self.hub.server.host, ] if self.debug_proxy: cmd.extend(['--log-level', 'debug']) if self.ssl_key: cmd.extend(['--ssl-key', self.ssl_key]) if self.ssl_cert: cmd.extend(['--ssl-cert', self.ssl_cert]) self.log.info("Starting proxy @ %s", self.proxy.public_server.url) self.log.debug("Proxy cmd: %s", cmd) try: self.proxy_process = Popen(cmd, env=env) except FileNotFoundError as e: self.log.error( "Failed to find proxy %r\n" "The proxy can be installed with `npm install -g configurable-http-proxy`" % self.proxy_cmd) self.exit(1) def _check(): status = self.proxy_process.poll() if status is not None: e = RuntimeError("Proxy failed to start with exit code %i" % status) # py2-compatible `raise e from None` e.__cause__ = None raise e for server in (self.proxy.public_server, self.proxy.api_server): for i in range(10): _check() try: yield server.wait_up(1) except TimeoutError: continue else: break yield server.wait_up(1) self.log.debug("Proxy started and appears to be up") @gen.coroutine def check_proxy(self): if self.proxy_process.poll() is None: return self.log.error( "Proxy stopped with exit code %r", 'unknown' if self.proxy_process is None else self.proxy_process.poll()) yield self.start_proxy() self.log.info("Setting up routes on new proxy") yield self.proxy.add_all_users() self.log.info("New proxy back up, and good to go") def init_tornado_settings(self): """Set up the tornado settings dict.""" base_url = self.hub.server.base_url template_path = os.path.join(self.data_files_path, 'templates'), jinja_env = Environment(loader=FileSystemLoader(template_path), **self.jinja_environment_options) login_url = self.authenticator.login_url(base_url) logout_url = self.authenticator.logout_url(base_url) # if running from git, disable caching of require.js # otherwise cache based on server start time parent = os.path.dirname(os.path.dirname(jupyterhub.__file__)) if os.path.isdir(os.path.join(parent, '.git')): version_hash = '' else: version_hash = datetime.now().strftime("%Y%m%d%H%M%S"), settings = dict( config=self.config, log=self.log, db=self.db, proxy=self.proxy, hub=self.hub, admin_users=self.admin_users, admin_access=self.admin_access, authenticator=self.authenticator, spawner_class=self.spawner_class, base_url=self.base_url, cookie_secret=self.cookie_secret, login_url=login_url, logout_url=logout_url, static_path=os.path.join(self.data_files_path, 'static'), static_url_prefix=url_path_join(self.hub.server.base_url, 'static/'), static_handler_class=CacheControlStaticFilesHandler, template_path=template_path, jinja2_env=jinja_env, version_hash=version_hash, ) # allow configured settings to have priority settings.update(self.tornado_settings) self.tornado_settings = settings def init_tornado_application(self): """Instantiate the tornado Application object""" self.tornado_application = web.Application(self.handlers, **self.tornado_settings) def write_pid_file(self): pid = os.getpid() if self.pid_file: self.log.debug("Writing PID %i to %s", pid, self.pid_file) with open(self.pid_file, 'w') as f: f.write('%i' % pid) @gen.coroutine @catch_config_error def initialize(self, *args, **kwargs): super().initialize(*args, **kwargs) if self.generate_config or self.subapp: return self.load_config_file(self.config_file) self.init_logging() if 'JupyterHubApp' in self.config: self.log.warn( "Use JupyterHub in config, not JupyterHubApp. Outdated config:\n%s", '\n'.join('JupyterHubApp.{key} = {value!r}'.format(key=key, value=value) for key, value in self.config.JupyterHubApp.items())) cfg = self.config.copy() cfg.JupyterHub.merge(cfg.JupyterHubApp) self.update_config(cfg) self.write_pid_file() self.init_ports() self.init_secrets() self.init_db() self.init_hub() self.init_proxy() yield self.init_users() self.init_handlers() self.init_tornado_settings() self.init_tornado_application() @gen.coroutine def cleanup(self): """Shutdown our various subprocesses and cleanup runtime files.""" futures = [] if self.cleanup_servers: self.log.info("Cleaning up single-user servers...") # request (async) process termination for user in self.db.query(orm.User): if user.spawner is not None: futures.append(user.stop()) else: self.log.info("Leaving single-user servers running") # clean up proxy while SUS are shutting down if self.cleanup_proxy: if self.proxy_process: self.log.info("Cleaning up proxy[%i]...", self.proxy_process.pid) if self.proxy_process.poll() is None: try: self.proxy_process.terminate() except Exception as e: self.log.error("Failed to terminate proxy process: %s", e) else: self.log.info("I didn't start the proxy, I can't clean it up") else: self.log.info("Leaving proxy running") # wait for the requests to stop finish: for f in futures: try: yield f except Exception as e: self.log.error("Failed to stop user: %s", e) self.db.commit() if self.pid_file and os.path.exists(self.pid_file): self.log.info("Cleaning up PID file %s", self.pid_file) os.remove(self.pid_file) # finally stop the loop once we are all cleaned up self.log.info("...done") def write_config_file(self): """Write our default config to a .py config file""" if os.path.exists(self.config_file) and not self.answer_yes: answer = '' def ask(): prompt = "Overwrite %s with default config? [y/N]" % self.config_file try: return input(prompt).lower() or 'n' except KeyboardInterrupt: print('') # empty line return 'n' answer = ask() while not answer.startswith(('y', 'n')): print("Please answer 'yes' or 'no'") answer = ask() if answer.startswith('n'): return config_text = self.generate_config_file() if isinstance(config_text, bytes): config_text = config_text.decode('utf8') print("Writing default config to: %s" % self.config_file) with open(self.config_file, mode='w') as f: f.write(config_text) @gen.coroutine def update_last_activity(self): """Update User.last_activity timestamps from the proxy""" routes = yield self.proxy.get_routes() for prefix, route in routes.items(): if 'user' not in route: # not a user route, ignore it continue user = orm.User.find(self.db, route['user']) if user is None: self.log.warn("Found no user for route: %s", route) continue try: dt = datetime.strptime(route['last_activity'], ISO8601_ms) except Exception: dt = datetime.strptime(route['last_activity'], ISO8601_s) user.last_activity = max(user.last_activity, dt) self.db.commit() yield self.proxy.check_routes(routes) @gen.coroutine def start(self): """Start the whole thing""" self.io_loop = loop = IOLoop.current() if self.subapp: self.subapp.start() loop.stop() return if self.generate_config: self.write_config_file() loop.stop() return # start the proxy try: yield self.start_proxy() except Exception as e: self.log.critical("Failed to start proxy", exc_info=True) self.exit(1) return loop.add_callback(self.proxy.add_all_users) if self.proxy_process: # only check / restart the proxy if we started it in the first place. # this means a restarted Hub cannot restart a Proxy that its # predecessor started. pc = PeriodicCallback(self.check_proxy, 1e3 * self.proxy_check_interval) pc.start() if self.last_activity_interval: pc = PeriodicCallback(self.update_last_activity, 1e3 * self.last_activity_interval) pc.start() # start the webserver self.http_server = tornado.httpserver.HTTPServer( self.tornado_application, xheaders=True) self.http_server.listen(self.hub_port) # register cleanup on both TERM and INT atexit.register(self.atexit) signal.signal(signal.SIGTERM, self.sigterm) def sigterm(self, signum, frame): self.log.critical("Received SIGTERM, shutting down") self.io_loop.stop() self.atexit() _atexit_ran = False def atexit(self): """atexit callback""" if self._atexit_ran: return self._atexit_ran = True # run the cleanup step (in a new loop, because the interrupted one is unclean) IOLoop.clear_current() loop = IOLoop() loop.make_current() loop.run_sync(self.cleanup) def stop(self): if not self.io_loop: return if self.http_server: self.io_loop.add_callback(self.http_server.stop) self.io_loop.add_callback(self.io_loop.stop) @gen.coroutine def launch_instance_async(self, argv=None): try: yield self.initialize(argv) yield self.start() except Exception as e: self.log.exception("") self.exit(1) @classmethod def launch_instance(cls, argv=None): self = cls.instance(argv=argv) loop = IOLoop.current() loop.add_callback(self.launch_instance_async, argv) try: loop.start() except KeyboardInterrupt: print("\nInterrupted")
class BytesTrait(HasTraits): value = Bytes(b'string')
class NotebookApp(BaseIPythonApplication): name = 'ipython-notebook' description = """ The IPython HTML Notebook. This launches a Tornado based HTML Notebook Server that serves up an HTML5/Javascript Notebook client. """ examples = _examples classes = IPythonConsoleApp.classes + [ MappingKernelManager, NotebookManager, FileNotebookManager ] flags = Dict(flags) aliases = Dict(aliases) kernel_argv = List(Unicode) def _log_level_default(self): return logging.INFO def _log_format_default(self): """override default log format to include time""" return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # file to be opened in the notebook server file_to_run = Unicode('') # Network related information. ip = Unicode(LOCALHOST, config=True, help="The IP address the notebook server will listen on.") def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8888, config=True, help="The port the notebook server will listen on.") port_retries = Integer( 50, config=True, help= "The number of additional ports to try if the specified port is not available." ) certfile = Unicode( u'', config=True, help="""The full path to an SSL/TLS certificate file.""") keyfile = Unicode( u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""") cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """) def _cookie_secret_default(self): return os.urandom(1024) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (NotebookApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) use_less = Bool(False, config=True, help="""Wether to use Browser Side less-css parsing instead of compiled css version in templates that allows it. This is mainly convenient when working on the less file to avoid a build step, or if user want to overwrite some of the less variables without having to recompile everything. You will need to install the less.js component in the static directory either in the source tree or in your profile folder. """) webapp_settings = Dict( config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") enable_mathjax = Bool( True, config=True, help="""Whether to enable MathJax for typesetting math/TeX MathJax is the javascript library IPython uses to render math/LaTeX. It is very large, so you may want to disable it if you have a slow internet connection, or for offline use of the notebook. When disabled, equations etc. will appear as their untransformed TeX source. """) def _enable_mathjax_changed(self, name, old, new): """set mathjax url to empty if mathjax is disabled""" if not new: self.mathjax_url = u'' base_project_url = Unicode('/', config=True, help='''The base URL for the notebook server. Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_project_url_changed(self, name, old, new): if not new.startswith('/'): self.base_project_url = '/' + new elif not new.endswith('/'): self.base_project_url = new + '/' base_kernel_url = Unicode('/', config=True, help='''The base URL for the kernel server Leading and trailing slashes can be omitted, and will automatically be added. ''') def _base_kernel_url_changed(self, name, old, new): if not new.startswith('/'): self.base_kernel_url = '/' + new elif not new.endswith('/'): self.base_kernel_url = new + '/' websocket_url = Unicode("", config=True, help="""The base URL for the websocket server, if it differs from the HTTP server (hint: it almost certainly doesn't). Should be in the form of an HTTP origin: ws[s]://hostname[:port] """) extra_static_paths = List( Unicode, config=True, help="""Extra paths to search for serving static files. This allows adding javascript/css to be available from the notebook server machine, or overriding individual files in the IPython""") def _extra_static_paths_default(self): return [os.path.join(self.profile_dir.location, 'static')] @property def static_file_path(self): """return extra paths + the default location""" return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""") def _mathjax_url_default(self): if not self.enable_mathjax: return u'' static_url_prefix = self.webapp_settings.get( "static_url_prefix", url_path_join(self.base_project_url, "static")) try: mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), self.static_file_path) except IOError: if self.certfile: # HTTPS: load from Rackspace CDN, because SSL certificate requires it base = u"https://c328740.ssl.cf1.rackcdn.com" else: base = u"http://cdn.mathjax.org" url = base + u"/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url else: self.log.info("Using local MathJax from %s" % mathjax) return url_path_join(static_url_prefix, u"mathjax/MathJax.js") def _mathjax_url_changed(self, name, old, new): if new and not self.enable_mathjax: # enable_mathjax=False overrides mathjax_url self.mathjax_url = u'' else: self.log.info("Using MathJax: %s", new) notebook_manager_class = DottedObjectName( 'IPython.html.services.notebooks.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') trust_xheaders = Bool( False, config=True, help= ("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Neccesary if the proxy handles SSL" )) def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) if argv is None: argv = sys.argv[1:] # Scrub frontend-specific flags self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags) # Kernel should inherit default config file from frontend self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) if self.extra_args: f = os.path.abspath(self.extra_args[0]) if os.path.isdir(f): nbdir = f else: self.file_to_run = f nbdir = os.path.dirname(f) self.config.NotebookManager.notebook_dir = nbdir def init_configurables(self): # force Session default to be secure default_secure(self.config) self.kernel_manager = MappingKernelManager( parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir=self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(parent=self, log=self.log) self.notebook_manager.load_notebook_names() self.cluster_manager = ClusterManager(parent=self, log=self.log) self.cluster_manager.update_profiles() def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False # hook up tornado 3's loggers to our app handlers for name in ('access', 'application', 'general'): logging.getLogger('tornado.%s' % name).handlers = self.log.handlers def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = NotebookWebApplication(self, self.kernel_manager, self.notebook_manager, self.cluster_manager, self.log, self.base_project_url, self.webapp_settings) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical( warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries + 1): try: self.http_server.listen(port, self.ip) except socket.error as e: # XXX: remove the e.errno == -9 block when we require # tornado >= 3.0 if e.errno == -9 and tornado.version_info[0] < 3: # The flags passed to socket.getaddrinfo from # tornado.netutils.bind_sockets can cause "gaierror: # [Errno -9] Address family for hostname not supported" # when the interface is not associated, for example. # Changing the flags to exclude socket.AI_ADDRCONFIG does # not cause this error, but the only way to do this is to # monkeypatch socket to remove the AI_ADDRCONFIG attribute saved_AI_ADDRCONFIG = socket.AI_ADDRCONFIG self.log.warn('Monkeypatching socket to fix tornado bug') del (socket.AI_ADDRCONFIG) try: # retry the tornado call without AI_ADDRCONFIG flags self.http_server.listen(port, self.ip) except socket.error as e2: e = e2 else: self.port = port success = True break # restore the monekypatch socket.AI_ADDRCONFIG = saved_AI_ADDRCONFIG if e.errno != errno.EADDRINUSE: raise self.log.info( 'The port %i is already in use, trying another random port.' % port) else: self.port = port success = True break if not success: self.log.critical( 'ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 time.sleep(0.1) info = self.log.info info('interrupted') print self.notebook_info() sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r, w, x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y'): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print "No answer for 5s:", print "resuming operation..." # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print self.notebook_info() def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn( "components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn( "components submodule unclean, you may see 404s on static/components" ) self.log.warn( "run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): self.init_logging() super(NotebookApp, self).initialize(argv) self.init_configurables() self.init_components() self.init_webapp() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" mgr_info = self.notebook_manager.info_string() + "\n" return mgr_info + "The IPython Notebook is running at: %s" % self._url def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" ip = self.ip if self.ip else '[all ip addresses on your system]' proto = 'https' if self.certfile else 'http' info = self.log.info self._url = "%s://%s:%i%s" % (proto, ip, self.port, self.base_project_url) for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) if self.open_browser or self.file_to_run: ip = self.ip or LOCALHOST try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if self.file_to_run: name, _ = os.path.splitext(os.path.basename(self.file_to_run)) url = self.notebook_manager.rev_mapping.get(name, '') else: url = '' if browser: b = lambda: browser.open("%s://%s:%i%s%s" % ( proto, ip, self.port, self.base_project_url, url), new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels()
class NotebookNotary(LoggingConfigurable): """A class for computing and verifying notebook signatures.""" profile_dir = Instance("IPython.core.profiledir.ProfileDir") def _profile_dir_default(self): from IPython.core.application import BaseIPythonApplication app = None try: if BaseIPythonApplication.initialized(): app = BaseIPythonApplication.instance() except MultipleInstanceError: pass if app is None: # create an app, without the global instance app = BaseIPythonApplication() app.initialize(argv=[]) return app.profile_dir db_file = Unicode(config=True, help="""The sqlite file in which to store notebook signatures. By default, this will be in your IPython profile. You can set it to ':memory:' to disable sqlite writing to the filesystem. """) def _db_file_default(self): if self.profile_dir is None: return ':memory:' return os.path.join(self.profile_dir.security_dir, u'nbsignatures.db') # 64k entries ~ 12MB cache_size = Integer(65535, config=True, help="""The number of notebook signatures to cache. When the number of signatures exceeds this value, the oldest 25% of signatures will be culled. """ ) db = Any() def _db_default(self): if sqlite3 is None: self.log.warn("Missing SQLite3, all notebooks will be untrusted!") return kwargs = dict(detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) db = sqlite3.connect(self.db_file, **kwargs) self.init_db(db) return db def init_db(self, db): db.execute(""" CREATE TABLE IF NOT EXISTS nbsignatures ( id integer PRIMARY KEY AUTOINCREMENT, algorithm text, signature text, path text, last_seen timestamp )""") db.execute(""" CREATE INDEX IF NOT EXISTS algosig ON nbsignatures(algorithm, signature) """) db.commit() algorithm = Enum(algorithms, default_value='sha256', config=True, help="""The hashing algorithm used to sign notebooks.""" ) def _algorithm_changed(self, name, old, new): self.digestmod = getattr(hashlib, self.algorithm) digestmod = Any() def _digestmod_default(self): return getattr(hashlib, self.algorithm) secret_file = Unicode(config=True, help="""The file where the secret key is stored.""" ) def _secret_file_default(self): if self.profile_dir is None: return '' return os.path.join(self.profile_dir.security_dir, 'notebook_secret') secret = Bytes(config=True, help="""The secret key with which notebooks are signed.""" ) def _secret_default(self): # note : this assumes an Application is running if os.path.exists(self.secret_file): with io.open(self.secret_file, 'rb') as f: return f.read() else: secret = base64.encodestring(os.urandom(1024)) self._write_secret_file(secret) return secret def _write_secret_file(self, secret): """write my secret to my secret_file""" self.log.info("Writing notebook-signing key to %s", self.secret_file) with io.open(self.secret_file, 'wb') as f: f.write(secret) try: os.chmod(self.secret_file, 0o600) except OSError: self.log.warn( "Could not set permissions on %s", self.secret_file ) return secret def compute_signature(self, nb): """Compute a notebook's signature by hashing the entire contents of the notebook via HMAC digest. """ hmac = HMAC(self.secret, digestmod=self.digestmod) # don't include the previous hash in the content to hash with signature_removed(nb): # sign the whole thing for b in yield_everything(nb): hmac.update(b) return hmac.hexdigest() def check_signature(self, nb): """Check a notebook's stored signature If a signature is stored in the notebook's metadata, a new signature is computed and compared with the stored value. Returns True if the signature is found and matches, False otherwise. The following conditions must all be met for a notebook to be trusted: - a signature is stored in the form 'scheme:hexdigest' - the stored scheme matches the requested scheme - the requested scheme is available from hashlib - the computed hash from notebook_signature matches the stored hash """ if nb.nbformat < 3: return False if self.db is None: return False signature = self.compute_signature(nb) r = self.db.execute("""SELECT id FROM nbsignatures WHERE algorithm = ? AND signature = ?; """, (self.algorithm, signature)).fetchone() if r is None: return False self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE algorithm = ? AND signature = ?; """, (datetime.utcnow(), self.algorithm, signature), ) self.db.commit() return True def sign(self, nb): """Sign a notebook, indicating that its output is trusted on this machine Stores hash algorithm and hmac digest in a local database of trusted notebooks. """ if nb.nbformat < 3: return signature = self.compute_signature(nb) self.store_signature(signature, nb) def store_signature(self, signature, nb): if self.db is None: return self.db.execute("""INSERT OR IGNORE INTO nbsignatures (algorithm, signature, last_seen) VALUES (?, ?, ?)""", (self.algorithm, signature, datetime.utcnow()) ) self.db.execute("""UPDATE nbsignatures SET last_seen = ? WHERE algorithm = ? AND signature = ?; """, (datetime.utcnow(), self.algorithm, signature), ) self.db.commit() n, = self.db.execute("SELECT Count(*) FROM nbsignatures").fetchone() if n > self.cache_size: self.cull_db() def unsign(self, nb): """Ensure that a notebook is untrusted by removing its signature from the trusted database, if present. """ signature = self.compute_signature(nb) self.db.execute("""DELETE FROM nbsignatures WHERE algorithm = ? AND signature = ?; """, (self.algorithm, signature) ) self.db.commit() def cull_db(self): """Cull oldest 25% of the trusted signatures when the size limit is reached""" self.db.execute("""DELETE FROM nbsignatures WHERE id IN ( SELECT id FROM nbsignatures ORDER BY last_seen DESC LIMIT -1 OFFSET ? ); """, (max(int(0.75 * self.cache_size), 1),)) def mark_cells(self, nb, trusted): """Mark cells as trusted if the notebook's signature can be verified Sets ``cell.metadata.trusted = True | False`` on all code cells, depending on whether the stored signature can be verified. This function is the inverse of check_cells """ if nb.nbformat < 3: return for cell in yield_code_cells(nb): cell['metadata']['trusted'] = trusted def _check_cell(self, cell, nbformat_version): """Do we trust an individual cell? Return True if: - cell is explicitly trusted - cell has no potentially unsafe rich output If a cell has no output, or only simple print statements, it will always be trusted. """ # explicitly trusted if cell['metadata'].pop("trusted", False): return True # explicitly safe output if nbformat_version >= 4: unsafe_output_types = ['execute_result', 'display_data'] safe_keys = {"output_type", "execution_count", "metadata"} else: # v3 unsafe_output_types = ['pyout', 'display_data'] safe_keys = {"output_type", "prompt_number", "metadata"} for output in cell['outputs']: output_type = output['output_type'] if output_type in unsafe_output_types: # if there are any data keys not in the safe whitelist output_keys = set(output) if output_keys.difference(safe_keys): return False return True def check_cells(self, nb): """Return whether all code cells are trusted If there are no code cells, return True. This function is the inverse of mark_cells. """ if nb.nbformat < 3: return False trusted = True for cell in yield_code_cells(nb): # only distrust a cell if it actually has some output to distrust if not self._check_cell(cell, nb.nbformat): trusted = False return trusted
class Comm(LoggingConfigurable): # If this is instantiated by a non-IPython kernel, shell will be None shell = Instance('IPython.core.interactiveshell.InteractiveShellABC', allow_none=True) kernel = Instance('IPython.kernel.zmq.kernelbase.Kernel') def _kernel_default(self): if Kernel.initialized(): return Kernel.instance() iopub_socket = Any() def _iopub_socket_default(self): return self.kernel.iopub_socket session = Instance('IPython.kernel.zmq.session.Session') def _session_default(self): if self.kernel is not None: return self.kernel.session target_name = Unicode('comm') topic = Bytes() def _topic_default(self): return ('comm-%s' % self.comm_id).encode('ascii') _open_data = Dict(help="data dict, if any, to be included in comm_open") _close_data = Dict(help="data dict, if any, to be included in comm_close") _msg_callback = Any() _close_callback = Any() _closed = Bool(True) comm_id = Unicode() def _comm_id_default(self): return uuid.uuid4().hex primary = Bool(True, help="Am I the primary or secondary Comm?") def __init__(self, target_name='', data=None, **kwargs): if target_name: kwargs['target_name'] = target_name super(Comm, self).__init__(**kwargs) if self.primary: # I am primary, open my peer. self.open(data) else: self._closed = False def _publish_msg(self, msg_type, data=None, metadata=None, buffers=None, **keys): """Helper for sending a comm message on IOPub""" data = {} if data is None else data metadata = {} if metadata is None else metadata content = json_clean(dict(data=data, comm_id=self.comm_id, **keys)) self.session.send( self.iopub_socket, msg_type, content, metadata=json_clean(metadata), parent=self.kernel._parent_header, ident=self.topic, buffers=buffers, ) def __del__(self): """trigger close on gc""" self.close() # publishing messages def open(self, data=None, metadata=None, buffers=None): """Open the frontend-side version of this comm""" if data is None: data = self._open_data comm_manager = getattr(self.kernel, 'comm_manager', None) if comm_manager is None: raise RuntimeError("Comms cannot be opened without a kernel " "and a comm_manager attached to that kernel.") comm_manager.register_comm(self) try: self._publish_msg('comm_open', data=data, metadata=metadata, buffers=buffers, target_name=self.target_name) self._closed = False except: comm_manager.unregister_comm(self) raise def close(self, data=None, metadata=None, buffers=None): """Close the frontend-side version of this comm""" if self._closed: # only close once return self._closed = True if data is None: data = self._close_data self._publish_msg( 'comm_close', data=data, metadata=metadata, buffers=buffers, ) self.kernel.comm_manager.unregister_comm(self) def send(self, data=None, metadata=None, buffers=None): """Send a message to the frontend-side version of this comm""" self._publish_msg( 'comm_msg', data=data, metadata=metadata, buffers=buffers, ) # registering callbacks def on_close(self, callback): """Register a callback for comm_close Will be called with the `data` of the close message. Call `on_close(None)` to disable an existing callback. """ self._close_callback = callback def on_msg(self, callback): """Register a callback for comm_msg Will be called with the `data` of any comm_msg messages. Call `on_msg(None)` to disable an existing callback. """ self._msg_callback = callback # handling of incoming messages def handle_close(self, msg): """Handle a comm_close message""" self.log.debug("handle_close[%s](%s)", self.comm_id, msg) if self._close_callback: self._close_callback(msg) def handle_msg(self, msg): """Handle a comm_msg message""" self.log.debug("handle_msg[%s](%s)", self.comm_id, msg) if self._msg_callback: if self.shell: self.shell.events.trigger('pre_execute') self._msg_callback(msg) if self.shell: self.shell.events.trigger('post_execute')
class NotebookNotary(LoggingConfigurable): """A class for computing and verifying notebook signatures.""" profile_dir = Instance("IPython.core.profiledir.ProfileDir") def _profile_dir_default(self): from IPython.core.application import BaseIPythonApplication app = None try: if BaseIPythonApplication.initialized(): app = BaseIPythonApplication.instance() except MultipleInstanceError: pass if app is None: # create an app, without the global instance app = BaseIPythonApplication() app.initialize(argv=[]) return app.profile_dir algorithm = Enum(algorithms, default_value='sha256', config=True, help="""The hashing algorithm used to sign notebooks.""") def _algorithm_changed(self, name, old, new): self.digestmod = getattr(hashlib, self.algorithm) digestmod = Any() def _digestmod_default(self): return getattr(hashlib, self.algorithm) secret_file = Unicode(config=True, help="""The file where the secret key is stored.""") def _secret_file_default(self): if self.profile_dir is None: return '' return os.path.join(self.profile_dir.security_dir, 'notebook_secret') secret = Bytes(config=True, help="""The secret key with which notebooks are signed.""") def _secret_default(self): # note : this assumes an Application is running if os.path.exists(self.secret_file): with io.open(self.secret_file, 'rb') as f: return f.read() else: secret = base64.encodestring(os.urandom(1024)) self._write_secret_file(secret) return secret def _write_secret_file(self, secret): """write my secret to my secret_file""" self.log.info("Writing notebook-signing key to %s", self.secret_file) with io.open(self.secret_file, 'wb') as f: f.write(secret) try: os.chmod(self.secret_file, 0o600) except OSError: self.log.warn("Could not set permissions on %s", self.secret_file) return secret def compute_signature(self, nb): """Compute a notebook's signature by hashing the entire contents of the notebook via HMAC digest. """ hmac = HMAC(self.secret, digestmod=self.digestmod) # don't include the previous hash in the content to hash with signature_removed(nb): # sign the whole thing for b in yield_everything(nb): hmac.update(b) return hmac.hexdigest() def check_signature(self, nb): """Check a notebook's stored signature If a signature is stored in the notebook's metadata, a new signature is computed and compared with the stored value. Returns True if the signature is found and matches, False otherwise. The following conditions must all be met for a notebook to be trusted: - a signature is stored in the form 'scheme:hexdigest' - the stored scheme matches the requested scheme - the requested scheme is available from hashlib - the computed hash from notebook_signature matches the stored hash """ if nb.nbformat < 3: return False stored_signature = nb['metadata'].get('signature', None) if not stored_signature \ or not isinstance(stored_signature, string_types) \ or ':' not in stored_signature: return False stored_algo, sig = stored_signature.split(':', 1) if self.algorithm != stored_algo: return False my_signature = self.compute_signature(nb) return my_signature == sig def sign(self, nb): """Sign a notebook, indicating that its output is trusted stores 'algo:hmac-hexdigest' in notebook.metadata.signature e.g. 'sha256:deadbeef123...' """ if nb.nbformat < 3: return signature = self.compute_signature(nb) nb['metadata']['signature'] = "%s:%s" % (self.algorithm, signature) def mark_cells(self, nb, trusted): """Mark cells as trusted if the notebook's signature can be verified Sets ``cell.metadata.trusted = True | False`` on all code cells, depending on whether the stored signature can be verified. This function is the inverse of check_cells """ if nb.nbformat < 3: return for cell in yield_code_cells(nb): cell['metadata']['trusted'] = trusted def _check_cell(self, cell, nbformat_version): """Do we trust an individual cell? Return True if: - cell is explicitly trusted - cell has no potentially unsafe rich output If a cell has no output, or only simple print statements, it will always be trusted. """ # explicitly trusted if cell['metadata'].pop("trusted", False): return True # explicitly safe output if nbformat_version >= 4: safe = {'text/plain', 'image/png', 'image/jpeg'} unsafe_output_types = ['execute_result', 'display_data'] safe_keys = {"output_type", "execution_count", "metadata"} else: # v3 safe = {'text', 'png', 'jpeg'} unsafe_output_types = ['pyout', 'display_data'] safe_keys = {"output_type", "prompt_number", "metadata"} for output in cell['outputs']: output_type = output['output_type'] if output_type in unsafe_output_types: # if there are any data keys not in the safe whitelist output_keys = set(output) if output_keys.difference(safe_keys): return False return True def check_cells(self, nb): """Return whether all code cells are trusted If there are no code cells, return True. This function is the inverse of mark_cells. """ if nb.nbformat < 3: return False trusted = True for cell in yield_code_cells(nb): # only distrust a cell if it actually has some output to distrust if not self._check_cell(cell, nb.nbformat): trusted = False return trusted
class ColaboratoryApp(BaseIPythonApplication): name = 'jupyter-colaboratory' description = """ The Jupyter Colaboratory. This launches a Tornado based HTML Server that can run local Jupyter kernels while storing the notebook files in Google Drive, supporting real-time collaborative editing of the notebooks. """ examples = _examples aliases = aliases flags = flags classes = [ KernelManager, ProfileDir, Session, MappingKernelManager, ] flags = Dict(flags) aliases = Dict(aliases) kernel_argv = List(Unicode) _log_formatter_cls = LogFormatter def _log_level_default(self): return logging.INFO def _log_datefmt_default(self): """Exclude date from default date format""" return "%H:%M:%S" # create requested profiles by default, if they don't exist: auto_create = Bool(True) # Network related information. ip = Unicode('127.0.0.1', config=True, help="The IP address the notebook server will listen on.") def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' port = Integer(8844, config=True, help="The port the notebook server will listen on.") port_retries = Integer( 50, config=True, help= "The number of additional ports to try if the specified port is not available." ) certfile = Unicode( u'', config=True, help="""The full path to an SSL/TLS certificate file.""") keyfile = Unicode( u'', config=True, help="""The full path to a private key file for usage with SSL/TLS.""") cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. Set it to a value in a config file to enable logins to persist across server sessions. Note: Cookie secrets should be kept private, do not share config files with cookie_secret stored in plaintext (you can read the value from a file). """) def _cookie_secret_default(self): return os.urandom(1024) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. To generate, type in a python/IPython shell: from IPython.lib import passwd; passwd() The string should be of the form type:salt:hashed-password. """) open_browser = Bool(True, config=True, help="""Whether to open in a browser after starting. The specific browser used is platform dependent and determined by the python standard library `webbrowser` module, unless it is overridden using the --browser (ColaboratoryApp.browser) configuration option. """) browser = Unicode(u'', config=True, help="""Specify what command to use to invoke a web browser when opening the notebook. If not specified, the default browser will be determined by the `webbrowser` standard library module, which allows setting of the BROWSER environment variable to override it. """) webapp_settings = Dict( config=True, help="Supply overrides for the tornado.web.Application that the " "IPython notebook uses.") jinja_environment_options = Dict( config=True, help="Supply extra arguments that will be passed to Jinja environment." ) notebook_manager_class = DottedObjectName( 'IPython.html.services.notebooks.filenbmanager.FileNotebookManager', config=True, help='The notebook manager class to use.') kernel_manager_class = DottedObjectName( 'IPython.html.services.kernels.kernelmanager.MappingKernelManager', config=True, help='The kernel manager class to use.') session_manager_class = DottedObjectName( 'IPython.html.services.sessions.sessionmanager.SessionManager', config=True, help='The session manager class to use.') trust_xheaders = Bool( False, config=True, help= ("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Necessary if the proxy handles SSL" )) info_file = Unicode() def _info_file_default(self): info_file = "nbserver-%s.json" % os.getpid() return os.path.join(self.profile_dir.security_dir, info_file) def init_kernel_argv(self): """construct the kernel arguments""" # Kernel should get *absolute* path to profile directory self.kernel_argv = ["--profile-dir", self.profile_dir.location] def init_configurables(self): # force Session default to be secure default_secure(self.config) kls = import_item(self.kernel_manager_class) self.kernel_manager = kls( parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir=self.profile_dir.security_dir, ) kls = import_item(self.notebook_manager_class) self.notebook_manager = kls(parent=self, log=self.log) kls = import_item(self.session_manager_class) self.session_manager = kls(parent=self, log=self.log) def init_logging(self): # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. self.log.propagate = False # hook up tornado 3's loggers to our app handlers logger = logging.getLogger('tornado') logger.propagate = True logger.parent = self.log logger.setLevel(self.log.level) def init_webapp(self): """initialize tornado webapp and httpserver""" self.web_app = ColaboratoryWebApplication( self, self.kernel_manager, self.notebook_manager, self.session_manager, self.log, self.webapp_settings, self.jinja_environment_options) if self.certfile: ssl_options = dict(certfile=self.certfile) if self.keyfile: ssl_options['keyfile'] = self.keyfile else: ssl_options = None self.web_app.password = self.password self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, xheaders=self.trust_xheaders) if not self.ip: warning = "WARNING: The notebook server is listening on all IP addresses" if ssl_options is None: self.log.critical(warning + " and not using encryption. This " "is not recommended.") if not self.password: self.log.critical( warning + " and not using authentication. " "This is highly insecure and not recommended.") success = None for port in random_ports(self.port, self.port_retries + 1): try: self.http_server.listen(port, self.ip) except socket.error as e: if e.errno == errno.EADDRINUSE: self.log.info( 'The port %i is already in use, trying another random port.' % port) continue elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)): self.log.warn("Permission to listen on port %i denied" % port) continue else: raise else: self.port = port success = True break if not success: self.log.critical( 'ERROR: the notebook server could not be started because ' 'no available port could be found.') self.exit(1) @property def display_url(self): ip = self.ip if self.ip else '[all ip addresses on your system]' return self._url(ip) @property def connection_url(self): ip = self.ip if self.ip else 'localhost' return self._url(ip) def _url(self, ip): proto = 'https' if self.certfile else 'http' return "%s://%s:%i" % (proto, ip, self.port) def init_signal(self): if not sys.platform.startswith('win'): signal.signal(signal.SIGINT, self._handle_sigint) signal.signal(signal.SIGTERM, self._signal_stop) if hasattr(signal, 'SIGUSR1'): # Windows doesn't support SIGUSR1 signal.signal(signal.SIGUSR1, self._signal_info) if hasattr(signal, 'SIGINFO'): # only on BSD-based systems signal.signal(signal.SIGINFO, self._signal_info) def _handle_sigint(self, sig, frame): """SIGINT handler spawns confirmation dialog""" # register more forceful signal handler for ^C^C case signal.signal(signal.SIGINT, self._signal_stop) # request confirmation dialog in bg thread, to avoid # blocking the App thread = threading.Thread(target=self._confirm_exit) thread.daemon = True thread.start() def _restore_sigint_handler(self): """callback for restoring original SIGINT handler""" signal.signal(signal.SIGINT, self._handle_sigint) def _confirm_exit(self): """confirm shutdown on ^C A second ^C, or answering 'y' within 5s will cause shutdown, otherwise original SIGINT handler will be restored. This doesn't work on Windows. """ info = self.log.info info('interrupted') print(self.notebook_info()) sys.stdout.write("Shutdown this notebook server (y/[n])? ") sys.stdout.flush() r, w, x = select.select([sys.stdin], [], [], 5) if r: line = sys.stdin.readline() if line.lower().startswith('y') and 'n' not in line.lower(): self.log.critical("Shutdown confirmed") ioloop.IOLoop.instance().stop() return else: print("No answer for 5s:", end=' ') print("resuming operation...") # no answer, or answer is no: # set it back to original SIGINT handler # use IOLoop.add_callback because signal.signal must be called # from main thread ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) def _signal_stop(self, sig, frame): self.log.critical("received signal %s, stopping", sig) ioloop.IOLoop.instance().stop() def _signal_info(self, sig, frame): print(self.notebook_info()) def init_components(self): """Check the components submodule, and warn if it's unclean""" status = submodule.check_submodule_status() if status == 'missing': self.log.warn( "components submodule missing, running `git submodule update`") submodule.update_submodules(submodule.ipython_parent()) elif status == 'unclean': self.log.warn( "components submodule unclean, you may see 404s on static/components" ) self.log.warn( "run `setup.py submodule` or `git submodule update` to update") @catch_config_error def initialize(self, argv=None): super(ColaboratoryApp, self).initialize(argv) self.init_logging() self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() self.init_signal() def cleanup_kernels(self): """Shutdown all kernels. The kernels will shutdown themselves when this process no longer exists, but explicit shutdown allows the KernelManagers to cleanup the connection files. """ self.log.info('Shutting down kernels') self.kernel_manager.shutdown_all() def notebook_info(self): "Return the current working directory and the server url information" info = self.notebook_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url def server_info(self): """Return a JSONable dict of information about this server.""" return { 'url': self.connection_url, 'hostname': self.ip if self.ip else 'localhost', 'port': self.port, 'secure': bool(self.certfile), } def write_server_info_file(self): """Write the result of server_info() to the JSON file info_file.""" with open(self.info_file, 'w') as f: json.dump(self.server_info(), f, indent=2) def remove_server_info_file(self): """Remove the nbserver-<pid>.json file created for this server. Ignores the error raised when the file has already been removed. """ try: os.unlink(self.info_file) except OSError as e: if e.errno != errno.ENOENT: raise def start(self): """ Start the IPython Notebook server app, after initialization This method takes no arguments so all configuration and initialization must be done prior to calling this method.""" if self.subapp is not None: return self.subapp.start() info = self.log.info for line in self.notebook_info().split("\n"): info(line) info( "Use Control-C to stop this server and shut down all kernels (twice to skip confirmation)." ) self.write_server_info_file() if self.open_browser: try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: self.log.warn('No web browser found: %s.' % e) browser = None if browser: b = lambda: browser.open(self.connection_url, new=2) threading.Thread(target=b).start() try: ioloop.IOLoop.instance().start() except KeyboardInterrupt: info("Interrupted...") finally: self.cleanup_kernels() self.remove_server_info_file()
class NotebookNotary(LoggingConfigurable): """A class for computing and verifying notebook signatures.""" profile_dir = Instance("IPython.core.profiledir.ProfileDir") def _profile_dir_default(self): from IPython.core.application import BaseIPythonApplication app = None try: if BaseIPythonApplication.initialized(): app = BaseIPythonApplication.instance() except MultipleInstanceError: pass if app is None: # create an app, without the global instance app = BaseIPythonApplication() app.initialize() return app.profile_dir algorithm = Enum(algorithms, default_value='sha256', config=True, help="""The hashing algorithm used to sign notebooks.""") def _algorithm_changed(self, name, old, new): self.digestmod = getattr(hashlib, self.algorithm) digestmod = Any() def _digestmod_default(self): return getattr(hashlib, self.algorithm) secret_file = Unicode() def _secret_file_default(self): if self.profile_dir is None: return '' return os.path.join(self.profile_dir.security_dir, 'notebook_secret') secret = Bytes(config=True, help="""The secret key with which notebooks are signed.""") def _secret_default(self): # note : this assumes an Application is running if os.path.exists(self.secret_file): with io.open(self.secret_file, 'rb') as f: return f.read() else: secret = base64.encodestring(os.urandom(1024)) self._write_secret_file(secret) return secret def _write_secret_file(self, secret): """write my secret to my secret_file""" self.log.info("Writing notebook-signing key to %s", self.secret_file) with io.open(self.secret_file, 'wb') as f: f.write(secret) try: os.chmod(self.secret_file, 0o600) except OSError: self.log.warn("Could not set permissions on %s", self.secret_file) return secret def compute_signature(self, nb): """Compute a notebook's signature by hashing the entire contents of the notebook via HMAC digest. """ hmac = HMAC(self.secret, digestmod=self.digestmod) # don't include the previous hash in the content to hash with signature_removed(nb): # sign the whole thing for b in yield_everything(nb): hmac.update(b) return hmac.hexdigest() def check_signature(self, nb): """Check a notebook's stored signature If a signature is stored in the notebook's metadata, a new signature is computed and compared with the stored value. Returns True if the signature is found and matches, False otherwise. The following conditions must all be met for a notebook to be trusted: - a signature is stored in the form 'scheme:hexdigest' - the stored scheme matches the requested scheme - the requested scheme is available from hashlib - the computed hash from notebook_signature matches the stored hash """ stored_signature = nb['metadata'].get('signature', None) if not stored_signature \ or not isinstance(stored_signature, string_types) \ or ':' not in stored_signature: return False stored_algo, sig = stored_signature.split(':', 1) if self.algorithm != stored_algo: return False my_signature = self.compute_signature(nb) return my_signature == sig def sign(self, nb): """Sign a notebook, indicating that its output is trusted stores 'algo:hmac-hexdigest' in notebook.metadata.signature e.g. 'sha256:deadbeef123...' """ signature = self.compute_signature(nb) nb['metadata']['signature'] = "%s:%s" % (self.algorithm, signature) def mark_cells(self, nb, trusted): """Mark cells as trusted if the notebook's signature can be verified Sets ``cell.trusted = True | False`` on all code cells, depending on whether the stored signature can be verified. This function is the inverse of check_cells """ if not nb['worksheets']: # nothing to mark if there are no cells return for cell in nb['worksheets'][0]['cells']: if cell['cell_type'] == 'code': cell['trusted'] = trusted def check_cells(self, nb): """Return whether all code cells are trusted If there are no code cells, return True. This function is the inverse of mark_cells. """ if not nb['worksheets']: return True for cell in nb['worksheets'][0]['cells']: if cell['cell_type'] != 'code': continue if not cell.get('trusted', False): return False return True
class Device(App): """Base class for MSMAccelerator devices. These are processes that request data from the server, and run on a ZMQ REQ port. Current subclasses include Simulator and Modeler. When the device boots up, it will send a message to the server with the msg_type 'register_{ClassName}', where ClassName is the name of the subclass of device that was intantiated. When it receives a return message, that the method on_startup_message() will be called. Note, if you want to interface directly with the ZMQ socket, it's just self.socket """ name = 'device' path = 'msmaccelerator.core.device.Device' short_description = 'Base class for MSMAccelerator devices' long_description = '''Contains common code for processes that connect to the msmaccelerator server to request data and do stuff''' zmq_port = Int(12345, config=True, help='ZeroMQ port to connect to the server on') zmq_url = Unicode('127.0.0.1', config=True, help='URL to connect to server with') uuid = Bytes(help='Unique identifier for this device') def _uuid_default(self): return str(uuid.uuid4()) aliases = dict(zmq_port='Device.zmq_port', zmq_url='Device.zmq_url') @property def zmq_connection_string(self): return 'tcp://%s:%s' % (self.zmq_url, self.zmq_port) def start(self): self.ctx = zmq.Context() self.socket = self.ctx.socket(zmq.REQ) # we're using the uuid to set the identity of the socket # AND we're going to put it explicitly inside of the header of # the messages we send. Within the actual bytes that go over the wire, # this means that the uuid will actually be printed twice, but # its not a big deal. It makes it easier for the server application # code to have the sender of each message, and for the message json # itself to be "complete", insted of having the sender in a separate # data structure. self.socket.setsockopt(zmq.IDENTITY, self.uuid) self.socket.connect(self.zmq_connection_string) # send the "here i am message" to the server, and receive a response # we're using a "robust" send/recv pattern, basically retrying the # request a fixed number of times if no response is heard from the # server msg = self.send_recv(msg_type='register_%s' % self.__class__.__name__) self.on_startup_message(msg) def on_startup_message(self, msg_type, msg): """This method is called when the device receives its startup message from the server """ raise NotImplementedError( 'This method should be overriden in a device subclass') def send_message(self, msg_type, content=None): """Send a message to the server asynchronously. Since we're using the request/reply pattern, after calling send you need to call recv to get the server's response. Consider instead using the send_recv method instead See Also -------- send_recv """ if content is None: content = {} self.socket.send_json( pack_message(msg_type=msg_type, content=content, sender_id=self.uuid)) def recv_message(self): """Receive a message from the server. Note, this methos is not async -- it blocks the device until the server delivers a message """ raw_msg = self.socket.recv() msg = yaml.load(raw_msg) return Message(msg) def send_recv(self, msg_type, content=None, timeout=10, retries=3): """Send a message to the server and receive a response This method inplementes the "Lazy-Pirate pattern" for relaible request/reply flows described in the ZeroMQ book. Parameters ---------- msg_type : str The type of the message to send. This is an essential part of the MSMAccelerator messaging protocol. content : dict The contents of the message to send. This is an essential part of the MSMAccelerator messaging protocol. timeout : int, float, default=3 The timeout, in seconds, to wait for a response from the server. retries : int, default=3 If a response from the server is not received within `timeout` seconds, we'll retry sending our payload at most `retries` number of times. After that point, if no return message has been received, we'll throw an IOError. """ timeout_ms = timeout * 1000 poller = zmq.Poller() for i in range(retries): self.send_message(msg_type, content) poller.register(self.socket, zmq.POLLIN) if poller.poll(timeout_ms): return self.recv_message() else: self.log.error( 'No response received from server on' 'msg_type=%s. Retrying...', msg_type) poller.unregister(self.socket) self.socket.close() self.socket = self.ctx.socket(zmq.REQ) self.socket.connect(self.zmq_connection_string) raise IOError('Network timeout. Server is unresponsive.')