class MappingKernelManager(MultiKernelManager): """A KernelManager that handles notebook mapping and HTTP error handling""" def _kernel_manager_class_default(self): return "IPython.kernel.ioloop.IOLoopKernelManager" kernel_argv = List(Unicode) #------------------------------------------------------------------------- # Methods for managing kernels and sessions #------------------------------------------------------------------------- def _handle_kernel_died(self, kernel_id): """notice that a kernel died""" self.log.warn("Kernel %s died, removing from map.", kernel_id) self.remove_kernel(kernel_id) def start_kernel(self, kernel_id=None, **kwargs): """Start a kernel for a session an return its kernel_id. Parameters ---------- kernel_id : uuid The uuid to associate the new kernel with. If this is not None, this kernel will be persistent whenever it is requested. """ if kernel_id is None: kwargs['extra_arguments'] = self.kernel_argv kernel_id = super(MappingKernelManager, self).start_kernel(**kwargs) self.log.info("Kernel started: %s" % kernel_id) self.log.debug("Kernel args: %r" % kwargs) # register callback for failed auto-restart self.add_restart_callback( kernel_id, lambda: self._handle_kernel_died(kernel_id), 'dead', ) else: self._check_kernel_id(kernel_id) self.log.info("Using existing kernel: %s" % kernel_id) return kernel_id def shutdown_kernel(self, kernel_id, now=False): """Shutdown a kernel by kernel_id""" self._check_kernel_id(kernel_id) super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now) def kernel_model(self, kernel_id, ws_url): """Return a dictionary of kernel information described in the JSON standard model.""" self._check_kernel_id(kernel_id) model = {"id": kernel_id, "ws_url": ws_url} return model def list_kernels(self, ws_url): """Returns a list of kernel_id's of kernels running.""" kernels = [] kernel_ids = super(MappingKernelManager, self).list_kernel_ids() for kernel_id in kernel_ids: model = self.kernel_model(kernel_id, ws_url) kernels.append(model) return kernels # override _check_kernel_id to raise 404 instead of KeyError def _check_kernel_id(self, kernel_id): """Check a that a kernel_id exists and raise 404 if not.""" if kernel_id not in self: raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
class InteractiveShellApp(Configurable): """A Mixin for applications that start InteractiveShell instances. Provides configurables for loading extensions and executing files as part of configuring a Shell environment. The following methods should be called by the :meth:`initialize` method of the subclass: - :meth:`init_path` - :meth:`init_shell` (to be implemented by the subclass) - :meth:`init_gui_pylab` - :meth:`init_extensions` - :meth:`init_code` """ extensions = List( Unicode, config=True, help="A list of dotted module names of IPython extensions to load.") extra_extension = Unicode( '', config=True, help="dotted module name of an IPython extension to load.") def _extra_extension_changed(self, name, old, new): if new: # add to self.extensions self.extensions.append(new) # Extensions that are always loaded (not configurable) default_extensions = List(Unicode, [u'storemagic'], config=False) exec_files = List(Unicode, config=True, help="""List of files to run at IPython startup.""") file_to_run = Unicode('', config=True, help="""A file to be run""") exec_lines = List(Unicode, config=True, help="""lines of code to run at IPython startup.""") code_to_run = Unicode('', config=True, help="Execute the given command string.") module_to_run = Unicode('', config=True, help="Run the module as a script.") gui = CaselessStrEnum( ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx'), config=True, help= "Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx')." ) pylab = CaselessStrEnum( ['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'], config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """) pylab_import_all = Bool( True, config=True, help="""If true, an 'import *' is done from numpy and pylab, when using pylab""") shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') def init_path(self): """Add current working directory, '', to sys.path""" if sys.path[0] != '': sys.path.insert(0, '') def init_shell(self): raise NotImplementedError("Override in subclasses") def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" if self.gui or self.pylab: shell = self.shell try: if self.pylab: gui, backend = pylabtools.find_gui_and_backend(self.pylab) self.log.info("Enabling GUI event loop integration, " "toolkit=%s, pylab=%s" % (gui, self.pylab)) shell.enable_pylab(gui, import_all=self.pylab_import_all, welcome_message=True) else: self.log.info("Enabling GUI event loop integration, " "toolkit=%s" % self.gui) shell.enable_gui(self.gui) except Exception: self.log.warn("GUI event loop or pylab initialization failed") self.shell.showtraceback() def init_extensions(self): """Load all IPython extensions in IPythonApp.extensions. This uses the :meth:`ExtensionManager.load_extensions` to load all the extensions listed in ``self.extensions``. """ try: self.log.debug("Loading IPython extensions...") extensions = self.default_extensions + self.extensions for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) self.shell.extension_manager.load_extension(ext) except: self.log.warn("Error in loading extension: %s" % ext + "\nCheck your config files in %s" % self.profile_dir.location) self.shell.showtraceback() except: self.log.warn("Unknown error in loading extensions:") self.shell.showtraceback() def init_code(self): """run the pre-flight code, specified via exec_lines""" self._run_startup_files() self._run_exec_lines() self._run_exec_files() self._run_cmd_line_code() self._run_module() # flush output, so itwon't be attached to the first cell sys.stdout.flush() sys.stderr.flush() # Hide variables defined here from %who etc. self.shell.user_ns_hidden.update(self.shell.user_ns) def _run_exec_lines(self): """Run lines of code in IPythonApp.exec_lines in the user's namespace.""" if not self.exec_lines: return try: self.log.debug("Running code from IPythonApp.exec_lines...") for line in self.exec_lines: try: self.log.info("Running code in user namespace: %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user " "namespace: %s" % line) self.shell.showtraceback() except: self.log.warn("Unknown error in handling IPythonApp.exec_lines:") self.shell.showtraceback() def _exec_file(self, fname): try: full_filename = filefind(fname, [u'.', self.ipython_dir]) except IOError as e: self.log.warn("File not found: %r" % fname) return # Make sure that the running script gets a proper sys.argv as if it # were run from a system shell. save_argv = sys.argv sys.argv = [full_filename] + self.extra_args[1:] # protect sys.argv from potential unicode strings on Python 2: if not py3compat.PY3: sys.argv = [py3compat.cast_bytes(a) for a in sys.argv] try: if os.path.isfile(full_filename): self.log.info("Running file in user namespace: %s" % full_filename) if full_filename.endswith('.ipy'): self.shell.safe_execfile_ipy(full_filename) else: # default to python, even without extension self.shell.safe_execfile(full_filename, self.shell.user_ns) finally: sys.argv = save_argv def _run_startup_files(self): """Run files from profile startup directory""" startup_dir = self.profile_dir.startup_dir startup_files = glob.glob(os.path.join(startup_dir, '*.py')) startup_files += glob.glob(os.path.join(startup_dir, '*.ipy')) if not startup_files: return self.log.debug("Running startup files from %s...", startup_dir) try: for fname in sorted(startup_files): self._exec_file(fname) except: self.log.warn("Unknown error in handling startup files:") self.shell.showtraceback() def _run_exec_files(self): """Run files from IPythonApp.exec_files""" if not self.exec_files: return self.log.debug("Running files in IPythonApp.exec_files...") try: for fname in self.exec_files: self._exec_file(fname) except: self.log.warn("Unknown error in handling IPythonApp.exec_files:") self.shell.showtraceback() def _run_cmd_line_code(self): """Run code or file specified at the command-line""" if self.code_to_run: line = self.code_to_run try: self.log.info("Running code given at command line (c=): %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user namespace: %s" % line) self.shell.showtraceback() # Like Python itself, ignore the second if the first of these is present elif self.file_to_run: fname = self.file_to_run try: self._exec_file(fname) except: self.log.warn("Error in executing file in user namespace: %s" % fname) self.shell.showtraceback() def _run_module(self): """Run module specified at the command-line.""" if self.module_to_run: # Make sure that the module gets a proper sys.argv as if it were # run using `python -m`. save_argv = sys.argv sys.argv = [sys.executable] + self.extra_args try: self.shell.safe_run_module(self.module_to_run, self.shell.user_ns) finally: sys.argv = save_argv
class MappingKernelManager(MultiKernelManager): """A KernelManager that handles notebok mapping and HTTP error handling""" kernel_argv = List(Unicode) time_to_dead = Float(3.0, config=True, help="""Kernel heartbeat interval in seconds.""") first_beat = Float(5.0, config=True, help="Delay (in seconds) before sending first heartbeat.") max_msg_size = Integer(65536, config=True, help=""" The max raw message size accepted from the browser over a WebSocket connection. """) _notebook_mapping = Dict() #------------------------------------------------------------------------- # Methods for managing kernels and sessions #------------------------------------------------------------------------- def kernel_for_notebook(self, notebook_id): """Return the kernel_id for a notebook_id or None.""" return self._notebook_mapping.get(notebook_id) def set_kernel_for_notebook(self, notebook_id, kernel_id): """Associate a notebook with a kernel.""" if notebook_id is not None: self._notebook_mapping[notebook_id] = kernel_id def notebook_for_kernel(self, kernel_id): """Return the notebook_id for a kernel_id or None.""" notebook_ids = [k for k, v in self._notebook_mapping.iteritems() if v == kernel_id] if len(notebook_ids) == 1: return notebook_ids[0] else: return None def delete_mapping_for_kernel(self, kernel_id): """Remove the kernel/notebook mapping for kernel_id.""" notebook_id = self.notebook_for_kernel(kernel_id) if notebook_id is not None: del self._notebook_mapping[notebook_id] def start_kernel(self, notebook_id=None): """Start a kernel for a notebok an return its kernel_id. Parameters ---------- notebook_id : uuid The uuid of the notebook to associate the new kernel with. If this is not None, this kernel will be persistent whenever the notebook requests a kernel. """ kernel_id = self.kernel_for_notebook(notebook_id) if kernel_id is None: kwargs = dict() kwargs['extra_arguments'] = self.kernel_argv kernel_id = super(MappingKernelManager, self).start_kernel(**kwargs) self.set_kernel_for_notebook(notebook_id, kernel_id) self.log.info("Kernel started: %s" % kernel_id) self.log.debug("Kernel args: %r" % kwargs) else: self.log.info("Using existing kernel: %s" % kernel_id) return kernel_id def kill_kernel(self, kernel_id): """Kill a kernel and remove its notebook association.""" self._check_kernel_id(kernel_id) super(MappingKernelManager, self).kill_kernel(kernel_id) self.delete_mapping_for_kernel(kernel_id) self.log.info("Kernel killed: %s" % kernel_id) def interrupt_kernel(self, kernel_id): """Interrupt a kernel.""" self._check_kernel_id(kernel_id) super(MappingKernelManager, self).interrupt_kernel(kernel_id) self.log.info("Kernel interrupted: %s" % kernel_id) def restart_kernel(self, kernel_id): """Restart a kernel while keeping clients connected.""" self._check_kernel_id(kernel_id) km = self.get_kernel(kernel_id) km.restart_kernel(now=True) self.log.info("Kernel restarted: %s" % kernel_id) return kernel_id # the following remains, in case the KM restart machinery is # somehow unacceptable # Get the notebook_id to preserve the kernel/notebook association. notebook_id = self.notebook_for_kernel(kernel_id) # Create the new kernel first so we can move the clients over. new_kernel_id = self.start_kernel() # Now kill the old kernel. self.kill_kernel(kernel_id) # Now save the new kernel/notebook association. We have to save it # after the old kernel is killed as that will delete the mapping. self.set_kernel_for_notebook(notebook_id, new_kernel_id) self.log.info("Kernel restarted: %s" % new_kernel_id) return new_kernel_id def create_iopub_stream(self, kernel_id): """Create a new iopub stream.""" self._check_kernel_id(kernel_id) return super(MappingKernelManager, self).create_iopub_stream(kernel_id) def create_shell_stream(self, kernel_id): """Create a new shell stream.""" self._check_kernel_id(kernel_id) return super(MappingKernelManager, self).create_shell_stream(kernel_id) def create_hb_stream(self, kernel_id): """Create a new hb stream.""" self._check_kernel_id(kernel_id) return super(MappingKernelManager, self).create_hb_stream(kernel_id) def _check_kernel_id(self, kernel_id): """Check a that a kernel_id exists and raise 404 if not.""" if kernel_id not in self: raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
class SSHLauncher(LocalProcessLauncher): """A minimal launcher for ssh. To be useful this will probably have to be extended to use the ``sshx`` idea for environment variables. There could be other things this needs as well. """ ssh_cmd = List(['ssh'], config=True, help="command for starting ssh") ssh_args = List(['-tt'], config=True, help="args to pass to ssh") scp_cmd = List(['scp'], config=True, help="command for sending files") program = List(['date'], help="Program to launch via ssh") program_args = List([], help="args to pass to remote program") hostname = Unicode('', config=True, help="hostname on which to launch the program") user = Unicode('', config=True, help="username for ssh") location = Unicode('', config=True, help="user@hostname location for ssh in one setting") to_fetch = List( [], config=True, help="List of (remote, local) files to fetch after starting") to_send = List( [], config=True, help="List of (local, remote) files to send before starting") def _hostname_changed(self, name, old, new): if self.user: self.location = u'%s@%s' % (self.user, new) else: self.location = new def _user_changed(self, name, old, new): self.location = u'%s@%s' % (new, self.hostname) def find_args(self): return self.ssh_cmd + self.ssh_args + [self.location] + \ list(map(pipes.quote, self.program + self.program_args)) def _send_file(self, local, remote): """send a single file""" remote = "%s:%s" % (self.location, remote) for i in range(10): if not os.path.exists(local): self.log.debug("waiting for %s" % local) time.sleep(1) else: break self.log.info("sending %s to %s", local, remote) check_output(self.scp_cmd + [local, remote]) def send_files(self): """send our files (called before start)""" if not self.to_send: return for local_file, remote_file in self.to_send: self._send_file(local_file, remote_file) def _fetch_file(self, remote, local): """fetch a single file""" full_remote = "%s:%s" % (self.location, remote) self.log.info("fetching %s from %s", local, full_remote) for i in range(10): # wait up to 10s for remote file to exist check = check_output(self.ssh_cmd + self.ssh_args + \ [self.location, 'test -e', remote, "&& echo 'yes' || echo 'no'"]) check = check.strip() if check == 'no': time.sleep(1) elif check == 'yes': break check_output(self.scp_cmd + [full_remote, local]) def fetch_files(self): """fetch remote files (called after start)""" if not self.to_fetch: return for remote_file, local_file in self.to_fetch: self._fetch_file(remote_file, local_file) def start(self, hostname=None, user=None): if hostname is not None: self.hostname = hostname if user is not None: self.user = user self.send_files() super(SSHLauncher, self).start() self.fetch_files() def signal(self, sig): if self.state == 'running': # send escaped ssh connection-closer self.process.stdin.write('~.') self.process.stdin.flush()
class IPEngineApp(BaseParallelApplication): name = 'ipengine' description = _description examples = _examples config_file_name = Unicode(default_config_file_name) classes = List([ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI]) startup_script = Unicode(u'', config=True, help='specify a script to be run at startup') startup_command = Unicode('', config=True, help='specify a command to be run at startup') url_file = Unicode(u'', config=True, help="""The full location of the file containing the connection information for the controller. If this is not given, the file must be in the security directory of the cluster directory. This location is resolved using the `profile` or `profile_dir` options.""", ) wait_for_url_file = Float(5, config=True, help="""The maximum number of seconds to wait for url_file to exist. This is useful for batch-systems and shared-filesystems where the controller and engine are started at the same time and it may take a moment for the controller to write the connector files.""") url_file_name = Unicode(u'ipcontroller-engine.json', config=True) def _cluster_id_changed(self, name, old, new): if new: base = 'ipcontroller-%s' % new else: base = 'ipcontroller' self.url_file_name = "%s-engine.json" % base log_url = Unicode('', config=True, help="""The URL for the iploggerapp instance, for forwarding logging to a central location.""") # an IPKernelApp instance, used to setup listening for shell frontends kernel_app = Instance(IPKernelApp) aliases = Dict(aliases) flags = Dict(flags) @property def kernel(self): """allow access to the Kernel object, so I look like IPKernelApp""" return self.engine.kernel def find_url_file(self): """Set the url file. Here we don't try to actually see if it exists for is valid as that is hadled by the connection logic. """ config = self.config # Find the actual controller key file if not self.url_file: self.url_file = os.path.join( self.profile_dir.security_dir, self.url_file_name ) def load_connector_file(self): """load config from a JSON connector file, at a *lower* priority than command-line/config files. """ self.log.info("Loading url_file %r", self.url_file) config = self.config with open(self.url_file) as f: d = json.loads(f.read()) # allow hand-override of location for disambiguation # and ssh-server try: config.EngineFactory.location except AttributeError: config.EngineFactory.location = d['location'] try: config.EngineFactory.sshserver except AttributeError: config.EngineFactory.sshserver = d.get('ssh') location = config.EngineFactory.location proto, ip = d['interface'].split('://') ip = disambiguate_ip_address(ip, location) d['interface'] = '%s://%s' % (proto, ip) # DO NOT allow override of basic URLs, serialization, or exec_key # JSON file takes top priority there config.Session.key = cast_bytes(d['exec_key']) config.EngineFactory.url = d['interface'] + ':%i' % d['registration'] config.Session.packer = d['pack'] config.Session.unpacker = d['unpack'] self.log.debug("Config changed:") self.log.debug("%r", config) self.connection_info = d def bind_kernel(self, **kwargs): """Promote engine to listening kernel, accessible to frontends.""" if self.kernel_app is not None: return self.log.info("Opening ports for direct connections as an IPython kernel") kernel = self.kernel kwargs.setdefault('config', self.config) kwargs.setdefault('log', self.log) kwargs.setdefault('profile_dir', self.profile_dir) kwargs.setdefault('session', self.engine.session) app = self.kernel_app = IPKernelApp(**kwargs) # allow IPKernelApp.instance(): IPKernelApp._instance = app app.init_connection_file() # relevant contents of init_sockets: app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port) app.log.debug("shell ROUTER Channel on port: %i", app.shell_port) app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port) app.log.debug("iopub PUB Channel on port: %i", app.iopub_port) kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER) app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port) app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port) # start the heartbeat, and log connection info: app.init_heartbeat() app.log_connection_info() app.write_connection_file() def init_engine(self): # This is the working dir by now. sys.path.insert(0, '') config = self.config # print config self.find_url_file() # was the url manually specified? keys = set(self.config.EngineFactory.keys()) keys = keys.union(set(self.config.RegistrationFactory.keys())) if keys.intersection(set(['ip', 'url', 'port'])): # Connection info was specified, don't wait for the file url_specified = True self.wait_for_url_file = 0 else: url_specified = False if self.wait_for_url_file and not os.path.exists(self.url_file): self.log.warn("url_file %r not found", self.url_file) self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file) tic = time.time() while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file): # wait for url_file to exist, or until time limit time.sleep(0.1) if os.path.exists(self.url_file): self.load_connector_file() elif not url_specified: self.log.fatal("Fatal: url file never arrived: %s", self.url_file) self.exit(1) try: exec_lines = config.IPKernelApp.exec_lines except AttributeError: try: exec_lines = config.InteractiveShellApp.exec_lines except AttributeError: exec_lines = config.IPKernelApp.exec_lines = [] try: exec_files = config.IPKernelApp.exec_files except AttributeError: try: exec_files = config.InteractiveShellApp.exec_files except AttributeError: exec_files = config.IPKernelApp.exec_files = [] if self.startup_script: exec_files.append(self.startup_script) if self.startup_command: exec_lines.append(self.startup_command) # Create the underlying shell class and Engine # shell_class = import_item(self.master_config.Global.shell_class) # print self.config try: self.engine = EngineFactory(config=config, log=self.log, connection_info=self.connection_info, ) except: self.log.error("Couldn't start the Engine", exc_info=True) self.exit(1) def forward_logging(self): if self.log_url: self.log.info("Forwarding logging to %s", self.log_url) context = self.engine.context lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = EnginePUBHandler(self.engine, lsock) handler.setLevel(self.log_level) self.log.addHandler(handler) def init_mpi(self): global mpi self.mpi = MPI(config=self.config) mpi_import_statement = self.mpi.init_script if mpi_import_statement: try: self.log.info("Initializing MPI:") self.log.info(mpi_import_statement) exec mpi_import_statement in globals() except: mpi = None else: mpi = None @catch_config_error def initialize(self, argv=None): super(IPEngineApp, self).initialize(argv) self.init_mpi() self.init_engine() self.forward_logging() def start(self): self.engine.start() try: self.engine.loop.start() except KeyboardInterrupt: self.log.critical("Engine Interrupted, shutting down...\n")
class Application(SingletonConfigurable): """A singleton application with full configuration support.""" # The name of the application, will usually match the name of the command # line application name = Unicode(u'application') # The description of the application that is printed at the beginning # of the help. description = Unicode(u'This is an application.') # default section descriptions option_description = Unicode(option_description) keyvalue_description = Unicode(keyvalue_description) subcommand_description = Unicode(subcommand_description) # The usage and example string that goes at the end of the help string. examples = Unicode() # A sequence of Configurable subclasses whose config=True attributes will # be exposed at the command line. classes = List([]) # The version string of this application. version = Unicode(u'0.0') # The log level for the application log_level = Enum( (0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'), default_value=logging.WARN, config=True, help="Set the log level by value or name.") def _log_level_changed(self, name, old, new): """Adjust the log level when log_level is set.""" if isinstance(new, basestring): new = getattr(logging, new) self.log_level = new self.log.setLevel(new) log_format = Unicode( "[%(name)s] %(message)s", config=True, help="The Logging format template", ) def _log_format_changed(self, name, old, new): """Change the log formatter when log_format is set.""" _log_handler = self.log.handlers[0] _log_formatter = logging.Formatter(new) _log_handler.setFormatter(_log_formatter) log = Instance(logging.Logger) def _log_default(self): """Start logging for this application. The default is to log to stderr using a StreamHandler, if no default handler already exists. The log level starts at logging.WARN, but this can be adjusted by setting the ``log_level`` attribute. """ log = logging.getLogger(self.__class__.__name__) log.setLevel(self.log_level) log.propagate = False _log = log # copied from Logger.hasHandlers() (new in Python 3.2) while _log: if _log.handlers: return log if not _log.propagate: break else: _log = _log.parent if sys.executable.endswith('pythonw.exe'): # this should really go to a file, but file-logging is only # hooked up in parallel applications _log_handler = logging.StreamHandler(open(os.devnull, 'w')) else: _log_handler = logging.StreamHandler() _log_formatter = logging.Formatter(self.log_format) _log_handler.setFormatter(_log_formatter) log.addHandler(_log_handler) return log # the alias map for configurables aliases = Dict({'log-level': 'Application.log_level'}) # flags for loading Configurables or store_const style flags # flags are loaded from this dict by '--key' flags # this must be a dict of two-tuples, the first element being the Config/dict # and the second being the help string for the flag flags = Dict() def _flags_changed(self, name, old, new): """ensure flags dict is valid""" for key, value in new.iteritems(): assert len(value) == 2, "Bad flag: %r:%s" % (key, value) assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s" % (key, value) assert isinstance(value[1], basestring), "Bad flag: %r:%s" % (key, value) # subcommands for launching other applications # if this is not empty, this will be a parent Application # this must be a dict of two-tuples, # the first element being the application class/import string # and the second being the help string for the subcommand subcommands = Dict() # parse_command_line will initialize a subapp, if requested subapp = Instance('IPython.config.application.Application', allow_none=True) # extra command-line arguments that don't set config values extra_args = List(Unicode) def __init__(self, **kwargs): SingletonConfigurable.__init__(self, **kwargs) # Ensure my class is in self.classes, so my attributes appear in command line # options and config files. if self.__class__ not in self.classes: self.classes.insert(0, self.__class__) def _config_changed(self, name, old, new): SingletonConfigurable._config_changed(self, name, old, new) self.log.debug('Config changed:') self.log.debug(repr(new)) @catch_config_error def initialize(self, argv=None): """Do the basic steps to configure me. Override in subclasses. """ self.parse_command_line(argv) def start(self): """Start the app mainloop. Override in subclasses. """ if self.subapp is not None: return self.subapp.start() def print_alias_help(self): """Print the alias part of the help.""" if not self.aliases: return lines = [] classdict = {} for cls in self.classes: # include all parents (up to, but excluding Configurable) in available names for c in cls.mro()[:-3]: classdict[c.__name__] = c for alias, longname in self.aliases.iteritems(): classname, traitname = longname.split('.', 1) cls = classdict[classname] trait = cls.class_traits(config=True)[traitname] help = cls.class_get_trait_help(trait).splitlines() # reformat first line help[0] = help[0].replace(longname, alias) + ' (%s)' % longname if len(alias) == 1: help[0] = help[0].replace('--%s=' % alias, '-%s ' % alias) lines.extend(help) # lines.append('') print os.linesep.join(lines) def print_flag_help(self): """Print the flag part of the help.""" if not self.flags: return lines = [] for m, (cfg, help) in self.flags.iteritems(): prefix = '--' if len(m) > 1 else '-' lines.append(prefix + m) lines.append(indent(dedent(help.strip()))) # lines.append('') print os.linesep.join(lines) def print_options(self): if not self.flags and not self.aliases: return lines = ['Options'] lines.append('-' * len(lines[0])) lines.append('') for p in wrap_paragraphs(self.option_description): lines.append(p) lines.append('') print os.linesep.join(lines) self.print_flag_help() self.print_alias_help() print def print_subcommands(self): """Print the subcommand part of the help.""" if not self.subcommands: return lines = ["Subcommands"] lines.append('-' * len(lines[0])) lines.append('') for p in wrap_paragraphs(self.subcommand_description): lines.append(p) lines.append('') for subc, (cls, help) in self.subcommands.iteritems(): lines.append(subc) if help: lines.append(indent(dedent(help.strip()))) lines.append('') print os.linesep.join(lines) def print_help(self, classes=False): """Print the help for each Configurable class in self.classes. If classes=False (the default), only flags and aliases are printed. """ self.print_subcommands() self.print_options() if classes: if self.classes: print "Class parameters" print "----------------" print for p in wrap_paragraphs(self.keyvalue_description): print p print for cls in self.classes: cls.class_print_help() print else: print "To see all available configurables, use `--help-all`" print def print_description(self): """Print the application description.""" for p in wrap_paragraphs(self.description): print p print def print_examples(self): """Print usage and examples. This usage string goes at the end of the command line help string and should contain examples of the application's usage. """ if self.examples: print "Examples" print "--------" print print indent(dedent(self.examples.strip())) print def print_version(self): """Print the version string.""" print self.version def update_config(self, config): """Fire the traits events when the config is updated.""" # Save a copy of the current config. newconfig = deepcopy(self.config) # Merge the new config into the current one. newconfig._merge(config) # Save the combined config as self.config, which triggers the traits # events. self.config = newconfig @catch_config_error def initialize_subcommand(self, subc, argv=None): """Initialize a subcommand with argv.""" subapp, help = self.subcommands.get(subc) if isinstance(subapp, basestring): subapp = import_item(subapp) # clear existing instances self.__class__.clear_instance() # instantiate self.subapp = subapp.instance() # and initialize subapp self.subapp.initialize(argv) def flatten_flags(self): """flatten flags and aliases, so cl-args override as expected. This prevents issues such as an alias pointing to InteractiveShell, but a config file setting the same trait in TerminalInteraciveShell getting inappropriate priority over the command-line arg. Only aliases with exactly one descendent in the class list will be promoted. """ # build a tree of classes in our list that inherit from a particular # it will be a dict by parent classname of classes in our list # that are descendents mro_tree = defaultdict(list) for cls in self.classes: clsname = cls.__name__ for parent in cls.mro()[1:-3]: # exclude cls itself and Configurable,HasTraits,object mro_tree[parent.__name__].append(clsname) # flatten aliases, which have the form: # { 'alias' : 'Class.trait' } aliases = {} for alias, cls_trait in self.aliases.iteritems(): cls, trait = cls_trait.split('.', 1) children = mro_tree[cls] if len(children) == 1: # exactly one descendent, promote alias cls = children[0] aliases[alias] = '.'.join([cls, trait]) # flatten flags, which are of the form: # { 'key' : ({'Cls' : {'trait' : value}}, 'help')} flags = {} for key, (flagdict, help) in self.flags.iteritems(): newflag = {} for cls, subdict in flagdict.iteritems(): children = mro_tree[cls] # exactly one descendent, promote flag section if len(children) == 1: cls = children[0] newflag[cls] = subdict flags[key] = (newflag, help) return flags, aliases @catch_config_error def parse_command_line(self, argv=None): """Parse the command line arguments.""" argv = sys.argv[1:] if argv is None else argv if argv and argv[0] == 'help': # turn `ipython help notebook` into `ipython notebook -h` argv = argv[1:] + ['-h'] if self.subcommands and len(argv) > 0: # we have subcommands, and one may have been specified subc, subargv = argv[0], argv[1:] if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands: # it's a subcommand, and *not* a flag or class parameter return self.initialize_subcommand(subc, subargv) # Arguments after a '--' argument are for the script IPython may be # about to run, not IPython iteslf. For arguments parsed here (help and # version), we want to only search the arguments up to the first # occurrence of '--', which we're calling interpreted_argv. try: interpreted_argv = argv[:argv.index('--')] except ValueError: interpreted_argv = argv if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')): self.print_description() self.print_help('--help-all' in interpreted_argv) self.print_examples() self.exit(0) if '--version' in interpreted_argv or '-V' in interpreted_argv: self.print_version() self.exit(0) # flatten flags&aliases, so cl-args get appropriate priority: flags, aliases = self.flatten_flags() loader = KVArgParseConfigLoader(argv=argv, aliases=aliases, flags=flags) config = loader.load_config() self.update_config(config) # store unparsed args in extra_args self.extra_args = loader.extra_args @catch_config_error def load_config_file(self, filename, path=None): """Load a .py based config file by filename and path.""" loader = PyFileConfigLoader(filename, path=path) try: config = loader.load_config() except ConfigFileNotFound: # problem finding the file, raise raise except Exception: # try to get the full filename, but it will be empty in the # unlikely event that the error raised before filefind finished filename = loader.full_filename or filename # problem while running the file self.log.error("Exception while loading config file %s", filename, exc_info=True) else: self.log.debug("Loaded config file: %s", loader.full_filename) self.update_config(config) def generate_config_file(self): """generate default config file from Configurables""" lines = ["# Configuration file for %s." % self.name] lines.append('') lines.append('c = get_config()') lines.append('') for cls in self.classes: lines.append(cls.class_config_section()) return '\n'.join(lines) def exit(self, exit_status=0): self.log.debug("Exiting application: %s" % self.name) sys.exit(exit_status)
class BatchSystemLauncher(BaseLauncher): """Launch an external process using a batch system. This class is designed to work with UNIX batch systems like PBS, LSF, GridEngine, etc. The overall model is that there are different commands like qsub, qdel, etc. that handle the starting and stopping of the process. This class also has the notion of a batch script. The ``batch_template`` attribute can be set to a string that is a template for the batch script. This template is instantiated using string formatting. Thus the template can use {n} fot the number of instances. Subclasses can add additional variables to the template dict. """ # Subclasses must fill these in. See PBSEngineSet submit_command = List( [''], config=True, help="The name of the command line program used to submit jobs.") delete_command = List( [''], config=True, help="The name of the command line program used to delete jobs.") job_id_regexp = CRegExp( '', config=True, help= """A regular expression used to get the job id from the output of the submit_command.""") batch_template = Unicode( '', config=True, help="The string that is the batch script template itself.") batch_template_file = Unicode( u'', config=True, help="The file that contains the batch template.") batch_file_name = Unicode( u'batch_script', config=True, help="The filename of the instantiated batch script.") queue = Unicode(u'', config=True, help="The PBS Queue.") def _queue_changed(self, name, old, new): self.context[name] = new n = Integer(1) _n_changed = _queue_changed # not configurable, override in subclasses # PBS Job Array regex job_array_regexp = CRegExp('') job_array_template = Unicode('') # PBS Queue regex queue_regexp = CRegExp('') queue_template = Unicode('') # The default batch template, override in subclasses default_template = Unicode('') # The full path to the instantiated batch script. batch_file = Unicode(u'') # the format dict used with batch_template: context = Dict() def _context_default(self): """load the default context with the default values for the basic keys because the _trait_changed methods only load the context if they are set to something other than the default value. """ return dict(n=1, queue=u'', profile_dir=u'', cluster_id=u'') # the Formatter instance for rendering the templates: formatter = Instance(EvalFormatter, (), {}) def find_args(self): return self.submit_command + [self.batch_file] def __init__(self, work_dir=u'.', config=None, **kwargs): super(BatchSystemLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs) self.batch_file = os.path.join(self.work_dir, self.batch_file_name) def parse_job_id(self, output): """Take the output of the submit command and return the job id.""" m = self.job_id_regexp.search(output) if m is not None: job_id = m.group() else: raise LauncherError("Job id couldn't be determined: %s" % output) self.job_id = job_id self.log.info('Job submitted with job id: %r', job_id) return job_id def write_batch_script(self, n): """Instantiate and write the batch script to the work_dir.""" self.n = n # first priority is batch_template if set if self.batch_template_file and not self.batch_template: # second priority is batch_template_file with open(self.batch_template_file) as f: self.batch_template = f.read() if not self.batch_template: # third (last) priority is default_template self.batch_template = self.default_template # add jobarray or queue lines to user-specified template # note that this is *only* when user did not specify a template. # print self.job_array_regexp.search(self.batch_template) if not self.job_array_regexp.search(self.batch_template): self.log.debug("adding job array settings to batch script") firstline, rest = self.batch_template.split('\n', 1) self.batch_template = u'\n'.join( [firstline, self.job_array_template, rest]) # print self.queue_regexp.search(self.batch_template) if self.queue and not self.queue_regexp.search( self.batch_template): self.log.debug("adding PBS queue settings to batch script") firstline, rest = self.batch_template.split('\n', 1) self.batch_template = u'\n'.join( [firstline, self.queue_template, rest]) script_as_string = self.formatter.format(self.batch_template, **self.context) self.log.debug('Writing batch script: %s', self.batch_file) with open(self.batch_file, 'w') as f: f.write(script_as_string) os.chmod(self.batch_file, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) def start(self, n): """Start n copies of the process using a batch system.""" self.log.debug("Starting %s: %r", self.__class__.__name__, self.args) # Here we save profile_dir in the context so they # can be used in the batch script template as {profile_dir} self.write_batch_script(n) output = check_output(self.args, env=os.environ) job_id = self.parse_job_id(output) self.notify_start(job_id) return job_id def stop(self): output = check_output(self.delete_command + [self.job_id], env=os.environ) self.notify_stop( dict(job_id=self.job_id, output=output)) # Pass the output of the kill cmd return output
class NbConvertApp(BaseIPythonApplication): """Application used to convert to and from notebook file type (*.ipynb)""" name = 'ipython-nbconvert' aliases = nbconvert_aliases flags = nbconvert_flags def _log_level_default(self): return logging.INFO def _classes_default(self): classes = [NbConvertBase, ProfileDir] for pkg in (exporters, transformers, writers): for name in dir(pkg): cls = getattr(pkg, name) if isinstance(cls, type) and issubclass(cls, Configurable): classes.append(cls) return classes description = Unicode( """This application is used to convert notebook files (*.ipynb) to various other formats. WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""") output_base = Unicode('', config=True, help='''overwrite base name use for output files. can only be use when converting one notebook at a time. ''') examples = Unicode(""" The simplest way to use nbconvert is > ipython nbconvert mynotebook.ipynb which will convert mynotebook.ipynb to the default format (probably HTML). You can specify the export format with `--to`. Options include {0} > ipython nbconvert --to latex mynotebook.ipnynb Both HTML and LaTeX support multiple output templates. LaTeX includes 'basic', 'book', and 'article'. HTML includes 'basic' and 'full'. You can specify the flavor of the format used. > ipython nbconvert --to html --template basic mynotebook.ipynb You can also pipe the output to stdout, rather than a file > ipython nbconvert mynotebook.ipynb --stdout A post-processor can be used to compile a PDF > ipython nbconvert mynotebook.ipynb --to latex --post PDF You can get (and serve) a Reveal.js-powered slideshow > ipython nbconvert myslides.ipynb --to slides --post serve Multiple notebooks can be given at the command line in a couple of different ways: > ipython nbconvert notebook*.ipynb > ipython nbconvert notebook1.ipynb notebook2.ipynb or you can specify the notebooks list in a config file, containing:: c.NbConvertApp.notebooks = ["my_notebook.ipynb"] > ipython nbconvert --config mycfg.py """.format(get_export_names())) # Writer specific variables writer = Instance('IPython.nbconvert.writers.base.WriterBase', help="""Instance of the writer class used to write the results of the conversion.""") writer_class = DottedObjectName('FilesWriter', config=True, help="""Writer class used to write the results of the conversion""") writer_aliases = { 'fileswriter': 'IPython.nbconvert.writers.files.FilesWriter', 'debugwriter': 'IPython.nbconvert.writers.debug.DebugWriter', 'stdoutwriter': 'IPython.nbconvert.writers.stdout.StdoutWriter' } writer_factory = Type() def _writer_class_changed(self, name, old, new): if new.lower() in self.writer_aliases: new = self.writer_aliases[new.lower()] self.writer_factory = import_item(new) # Post-processor specific variables post_processor = Instance( 'IPython.nbconvert.post_processors.base.PostProcessorBase', help="""Instance of the PostProcessor class used to write the results of the conversion.""") post_processor_class = DottedOrNone( config=True, help="""PostProcessor class used to write the results of the conversion""") post_processor_aliases = { 'pdf': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor', 'serve': 'IPython.nbconvert.post_processors.serve.ServePostProcessor' } post_processor_factory = Type() def _post_processor_class_changed(self, name, old, new): if new.lower() in self.post_processor_aliases: new = self.post_processor_aliases[new.lower()] if new: self.post_processor_factory = import_item(new) # Other configurable variables export_format = CaselessStrEnum(get_export_names(), default_value="html", config=True, help="""The export format to be used.""") notebooks = List([], config=True, help="""List of notebooks to convert. Wildcards are supported. Filenames passed positionally will be added to the list. """) @catch_config_error def initialize(self, argv=None): super(NbConvertApp, self).initialize(argv) self.init_syspath() self.init_notebooks() self.init_writer() self.init_post_processor() def init_syspath(self): """ Add the cwd to the sys.path ($PYTHONPATH) """ sys.path.insert(0, os.getcwd()) def init_notebooks(self): """Construct the list of notebooks. If notebooks are passed on the command-line, they override notebooks specified in config files. Glob each notebook to replace notebook patterns with filenames. """ # Specifying notebooks on the command-line overrides (rather than adds) # the notebook list if self.extra_args: patterns = self.extra_args else: patterns = self.notebooks # Use glob to replace all the notebook patterns with filenames. filenames = [] for pattern in patterns: # Use glob to find matching filenames. Allow the user to convert # notebooks without having to type the extension. globbed_files = glob.glob(pattern) globbed_files.extend(glob.glob(pattern + '.ipynb')) if not globbed_files: self.log.warn("pattern %r matched no files", pattern) for filename in globbed_files: if not filename in filenames: filenames.append(filename) self.notebooks = filenames def init_writer(self): """ Initialize the writer (which is stateless) """ self._writer_class_changed(None, self.writer_class, self.writer_class) self.writer = self.writer_factory(parent=self) def init_post_processor(self): """ Initialize the post_processor (which is stateless) """ self._post_processor_class_changed(None, self.post_processor_class, self.post_processor_class) if self.post_processor_factory: self.post_processor = self.post_processor_factory(parent=self) def start(self): """ Ran after initialization completed """ super(NbConvertApp, self).start() self.convert_notebooks() def convert_notebooks(self): """ Convert the notebooks in the self.notebook traitlet """ # Export each notebook conversion_success = 0 if self.output_base != '' and len(self.notebooks) > 1: self.log.error( """UsageError: --output flag or `NbConvertApp.output_base` config option cannot be used when converting multiple notebooks. """) self.exit(1) exporter = exporter_map[self.export_format](config=self.config) for notebook_filename in self.notebooks: self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format) # Get a unique key for the notebook and set it in the resources object. basename = os.path.basename(notebook_filename) notebook_name = basename[:basename.rfind('.')] if self.output_base: notebook_name = self.output_base resources = {} resources['unique_key'] = notebook_name resources['output_files_dir'] = '%s_files' % notebook_name self.log.info("Support files will be in %s", os.path.join(resources['output_files_dir'], '')) # Try to export try: output, resources = exporter.from_filename(notebook_filename, resources=resources) except ConversionException as e: self.log.error("Error while converting '%s'", notebook_filename, exc_info=True) self.exit(1) else: write_resultes = self.writer.write(output, resources, notebook_name=notebook_name) #Post-process if post processor has been defined. if hasattr(self, 'post_processor') and self.post_processor: self.post_processor(write_resultes) conversion_success += 1 # If nothing was converted successfully, help the user. if conversion_success == 0: self.print_help() sys.exit(-1)
class InteractiveShellApp(Configurable): """A Mixin for applications that start InteractiveShell instances. Provides configurables for loading extensions and executing files as part of configuring a Shell environment. The following methods should be called by the :meth:`initialize` method of the subclass: - :meth:`init_path` - :meth:`init_shell` (to be implemented by the subclass) - :meth:`init_gui_pylab` - :meth:`init_extensions` - :meth:`init_code` """ extensions = List( Unicode, config=True, help="A list of dotted module names of IPython extensions to load.") extra_extension = Unicode( '', config=True, help="dotted module name of an IPython extension to load.") def _extra_extension_changed(self, name, old, new): if new: # add to self.extensions self.extensions.append(new) # Extensions that are always loaded (not configurable) default_extensions = List(Unicode, [u'storemagic'], config=False) hide_initial_ns = Bool( True, config=True, help= """Should variables loaded at startup (by startup files, exec_lines, etc.) be hidden from tools like %who?""") exec_files = List(Unicode, config=True, help="""List of files to run at IPython startup.""") exec_PYTHONSTARTUP = Bool( True, config=True, help="""Run the file referenced by the PYTHONSTARTUP environment variable at IPython startup.""") file_to_run = Unicode('', config=True, help="""A file to be run""") exec_lines = List(Unicode, config=True, help="""lines of code to run at IPython startup.""") code_to_run = Unicode('', config=True, help="Execute the given command string.") module_to_run = Unicode('', config=True, help="Run the module as a script.") gui = CaselessStrEnum( gui_keys, config=True, help="Enable GUI event loop integration with any of {0}.".format( gui_keys)) matplotlib = CaselessStrEnum( backend_keys, config=True, help="""Configure matplotlib for interactive use with the default matplotlib backend.""") pylab = CaselessStrEnum( backend_keys, config=True, help="""Pre-load matplotlib and numpy for interactive use, selecting a particular matplotlib backend and loop integration. """) pylab_import_all = Bool( True, config=True, help= """If true, IPython will populate the user namespace with numpy, pylab, etc. and an ``import *`` is done from numpy and pylab, when using pylab mode. When False, pylab mode should not import any names into the user namespace. """) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() def init_path(self): """Add current working directory, '', to sys.path""" if sys.path[0] != '': sys.path.insert(0, '') def init_shell(self): raise NotImplementedError("Override in subclasses") def init_gui_pylab(self): """Enable GUI event loop integration, taking pylab into account.""" enable = False shell = self.shell if self.pylab: enable = lambda key: shell.enable_pylab( key, import_all=self.pylab_import_all) key = self.pylab elif self.matplotlib: enable = shell.enable_matplotlib key = self.matplotlib elif self.gui: enable = shell.enable_gui key = self.gui if not enable: return try: r = enable(key) except ImportError: self.log.warn( "Eventloop or matplotlib integration failed. Is matplotlib installed?" ) self.shell.showtraceback() return except Exception: self.log.warn("GUI event loop or pylab initialization failed") self.shell.showtraceback() return if isinstance(r, tuple): gui, backend = r[:2] self.log.info( "Enabling GUI event loop integration, " "eventloop=%s, matplotlib=%s", gui, backend) if key == "auto": print("Using matplotlib backend: %s" % backend) else: gui = r self.log.info( "Enabling GUI event loop integration, " "eventloop=%s", gui) def init_extensions(self): """Load all IPython extensions in IPythonApp.extensions. This uses the :meth:`ExtensionManager.load_extensions` to load all the extensions listed in ``self.extensions``. """ try: self.log.debug("Loading IPython extensions...") extensions = self.default_extensions + self.extensions for ext in extensions: try: self.log.info("Loading IPython extension: %s" % ext) self.shell.extension_manager.load_extension(ext) except: self.log.warn("Error in loading extension: %s" % ext + "\nCheck your config files in %s" % self.profile_dir.location) self.shell.showtraceback() except: self.log.warn("Unknown error in loading extensions:") self.shell.showtraceback() def init_code(self): """run the pre-flight code, specified via exec_lines""" self._run_startup_files() self._run_exec_lines() self._run_exec_files() # Hide variables defined here from %who etc. if self.hide_initial_ns: self.shell.user_ns_hidden.update(self.shell.user_ns) # command-line execution (ipython -i script.py, ipython -m module) # should *not* be excluded from %whos self._run_cmd_line_code() self._run_module() # flush output, so itwon't be attached to the first cell sys.stdout.flush() sys.stderr.flush() def _run_exec_lines(self): """Run lines of code in IPythonApp.exec_lines in the user's namespace.""" if not self.exec_lines: return try: self.log.debug("Running code from IPythonApp.exec_lines...") for line in self.exec_lines: try: self.log.info("Running code in user namespace: %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user " "namespace: %s" % line) self.shell.showtraceback() except: self.log.warn("Unknown error in handling IPythonApp.exec_lines:") self.shell.showtraceback() def _exec_file(self, fname): try: full_filename = filefind(fname, [u'.', self.ipython_dir]) except IOError as e: self.log.warn("File not found: %r" % fname) return # Make sure that the running script gets a proper sys.argv as if it # were run from a system shell. save_argv = sys.argv sys.argv = [full_filename] + self.extra_args[1:] # protect sys.argv from potential unicode strings on Python 2: if not py3compat.PY3: sys.argv = [py3compat.cast_bytes(a) for a in sys.argv] try: if os.path.isfile(full_filename): self.log.info("Running file in user namespace: %s" % full_filename) # Ensure that __file__ is always defined to match Python # behavior. with preserve_keys(self.shell.user_ns, '__file__'): self.shell.user_ns['__file__'] = fname if full_filename.endswith('.ipy'): self.shell.safe_execfile_ipy(full_filename) else: # default to python, even without extension self.shell.safe_execfile(full_filename, self.shell.user_ns) finally: sys.argv = save_argv def _run_startup_files(self): """Run files from profile startup directory""" startup_dir = self.profile_dir.startup_dir startup_files = [] if self.exec_PYTHONSTARTUP and os.environ.get('PYTHONSTARTUP', False) and \ not (self.file_to_run or self.code_to_run or self.module_to_run): python_startup = os.environ['PYTHONSTARTUP'] self.log.debug("Running PYTHONSTARTUP file %s...", python_startup) try: self._exec_file(python_startup) except: self.log.warn( "Unknown error in handling PYTHONSTARTUP file %s:", python_startup) self.shell.showtraceback() finally: # Many PYTHONSTARTUP files set up the readline completions, # but this is often at odds with IPython's own completions. # Do not allow PYTHONSTARTUP to set up readline. if self.shell.has_readline: self.shell.set_readline_completer() startup_files += glob.glob(os.path.join(startup_dir, '*.py')) startup_files += glob.glob(os.path.join(startup_dir, '*.ipy')) if not startup_files: return self.log.debug("Running startup files from %s...", startup_dir) try: for fname in sorted(startup_files): self._exec_file(fname) except: self.log.warn("Unknown error in handling startup files:") self.shell.showtraceback() def _run_exec_files(self): """Run files from IPythonApp.exec_files""" if not self.exec_files: return self.log.debug("Running files in IPythonApp.exec_files...") try: for fname in self.exec_files: self._exec_file(fname) except: self.log.warn("Unknown error in handling IPythonApp.exec_files:") self.shell.showtraceback() def _run_cmd_line_code(self): """Run code or file specified at the command-line""" if self.code_to_run: line = self.code_to_run try: self.log.info("Running code given at command line (c=): %s" % line) self.shell.run_cell(line, store_history=False) except: self.log.warn("Error in executing line in user namespace: %s" % line) self.shell.showtraceback() # Like Python itself, ignore the second if the first of these is present elif self.file_to_run: fname = self.file_to_run try: self._exec_file(fname) except: self.log.warn("Error in executing file in user namespace: %s" % fname) self.shell.showtraceback() def _run_module(self): """Run module specified at the command-line.""" if self.module_to_run: # Make sure that the module gets a proper sys.argv as if it were # run using `python -m`. save_argv = sys.argv sys.argv = [sys.executable] + self.extra_args try: self.shell.safe_run_module(self.module_to_run, self.shell.user_ns) finally: sys.argv = save_argv
class DOMWidget(Widget): visible = Bool(True, help="Whether the widget is visible.", sync=True) _css = List( sync=True) # Internal CSS property list: (selector, key, value) def get_css(self, key, selector=""): """Get a CSS property of the widget. Note: This function does not actually request the CSS from the front-end; Only properties that have been set with set_css can be read. Parameters ---------- key: unicode CSS key selector: unicode (optional) JQuery selector used when the CSS key/value was set. """ if selector in self._css and key in self._css[selector]: return self._css[selector][key] else: return None def set_css(self, dict_or_key, value=None, selector=''): """Set one or more CSS properties of the widget. This function has two signatures: - set_css(css_dict, selector='') - set_css(key, value, selector='') Parameters ---------- css_dict : dict CSS key/value pairs to apply key: unicode CSS key value: CSS value selector: unicode (optional, kwarg only) JQuery selector to use to apply the CSS key/value. If no selector is provided, an empty selector is used. An empty selector makes the front-end try to apply the css to a default element. The default element is an attribute unique to each view, which is a DOM element of the view that should be styled with common CSS (see `$el_to_style` in the Javascript code). """ if value is None: css_dict = dict_or_key else: css_dict = {dict_or_key: value} for (key, value) in css_dict.items(): # First remove the selector/key pair from the css list if it exists. # Then add the selector/key pair and new value to the bottom of the # list. self._css = [ x for x in self._css if not (x[0] == selector and x[1] == key) ] self._css += [(selector, key, value)] self.send_state('_css') def add_class(self, class_names, selector=""): """Add class[es] to a DOM element. Parameters ---------- class_names: unicode or list Class name(s) to add to the DOM element(s). selector: unicode (optional) JQuery selector to select the DOM element(s) that the class(es) will be added to. """ class_list = class_names if isinstance(class_list, (list, tuple)): class_list = ' '.join(class_list) self.send({ "msg_type": "add_class", "class_list": class_list, "selector": selector }) def remove_class(self, class_names, selector=""): """Remove class[es] from a DOM element. Parameters ---------- class_names: unicode or list Class name(s) to remove from the DOM element(s). selector: unicode (optional) JQuery selector to select the DOM element(s) that the class(es) will be removed from. """ class_list = class_names if isinstance(class_list, (list, tuple)): class_list = ' '.join(class_list) self.send({ "msg_type": "remove_class", "class_list": class_list, "selector": selector, })
class Widget(LoggingConfigurable): #------------------------------------------------------------------------- # Class attributes #------------------------------------------------------------------------- _widget_construction_callback = None widgets = {} @staticmethod def on_widget_constructed(callback): """Registers a callback to be called when a widget is constructed. The callback must have the following signature: callback(widget)""" Widget._widget_construction_callback = callback @staticmethod def _call_widget_constructed(widget): """Static method, called when a widget is constructed.""" if Widget._widget_construction_callback is not None and callable( Widget._widget_construction_callback): Widget._widget_construction_callback(widget) #------------------------------------------------------------------------- # Traits #------------------------------------------------------------------------- _model_name = Unicode('WidgetModel', help="""Name of the backbone model registered in the front-end to create and sync this widget with.""") _view_name = Unicode(help="""Default view registered in the front-end to use to represent the widget.""", sync=True) _comm = Instance('IPython.kernel.comm.Comm') msg_throttle = Int(3, sync=True, help="""Maximum number of msgs the front-end can send before receiving an idle msg from the back-end.""") keys = List() def _keys_default(self): return [name for name in self.traits(sync=True)] _property_lock = Tuple((None, None)) _display_callbacks = Instance(CallbackDispatcher, ()) _msg_callbacks = Instance(CallbackDispatcher, ()) #------------------------------------------------------------------------- # (Con/de)structor #------------------------------------------------------------------------- def __init__(self, **kwargs): """Public constructor""" super(Widget, self).__init__(**kwargs) self.on_trait_change(self._handle_property_changed, self.keys) Widget._call_widget_constructed(self) def __del__(self): """Object disposal""" self.close() #------------------------------------------------------------------------- # Properties #------------------------------------------------------------------------- @property def comm(self): """Gets the Comm associated with this widget. If a Comm doesn't exist yet, a Comm will be created automagically.""" if self._comm is None: # Create a comm. self._comm = Comm(target_name=self._model_name) self._comm.on_msg(self._handle_msg) self._comm.on_close(self._close) Widget.widgets[self.model_id] = self # first update self.send_state() return self._comm @property def model_id(self): """Gets the model id of this widget. If a Comm doesn't exist yet, a Comm will be created automagically.""" return self.comm.comm_id #------------------------------------------------------------------------- # Methods #------------------------------------------------------------------------- def _close(self): """Private close - cleanup objects, registry entries""" del Widget.widgets[self.model_id] self._comm = None def close(self): """Close method. Closes the widget which closes the underlying comm. When the comm is closed, all of the widget views are automatically removed from the front-end.""" if self._comm is not None: self._comm.close() self._close() def send_state(self, key=None): """Sends the widget state, or a piece of it, to the front-end. Parameters ---------- key : unicode (optional) A single property's name to sync with the front-end. """ self._send({"method": "update", "state": self.get_state()}) def get_state(self, key=None): """Gets the widget state, or a piece of it. Parameters ---------- key : unicode (optional) A single property's name to get. """ keys = self.keys if key is None else [key] return {k: self._pack_widgets(getattr(self, k)) for k in keys} def send(self, content): """Sends a custom msg to the widget model in the front-end. Parameters ---------- content : dict Content of the message to send. """ self._send({"method": "custom", "content": content}) def on_msg(self, callback, remove=False): """(Un)Register a custom msg receive callback. Parameters ---------- callback: callable callback will be passed two arguments when a message arrives:: callback(widget, content) remove: bool True if the callback should be unregistered.""" self._msg_callbacks.register_callback(callback, remove=remove) def on_displayed(self, callback, remove=False): """(Un)Register a widget displayed callback. Parameters ---------- callback: method handler Must have a signature of:: callback(widget, **kwargs) kwargs from display are passed through without modification. remove: bool True if the callback should be unregistered.""" self._display_callbacks.register_callback(callback, remove=remove) #------------------------------------------------------------------------- # Support methods #------------------------------------------------------------------------- @contextmanager def _lock_property(self, key, value): """Lock a property-value pair. NOTE: This, in addition to the single lock for all state changes, is flawed. In the future we may want to look into buffering state changes back to the front-end.""" self._property_lock = (key, value) try: yield finally: self._property_lock = (None, None) def _should_send_property(self, key, value): """Check the property lock (property_lock)""" return key != self._property_lock[0] or \ value != self._property_lock[1] # Event handlers @_show_traceback def _handle_msg(self, msg): """Called when a msg is received from the front-end""" data = msg['content']['data'] method = data['method'] if not method in ['backbone', 'custom']: self.log.error( 'Unknown front-end to back-end widget msg with method "%s"' % method) # Handle backbone sync methods CREATE, PATCH, and UPDATE all in one. if method == 'backbone' and 'sync_data' in data: sync_data = data['sync_data'] self._handle_receive_state(sync_data) # handles all methods # Handle a custom msg from the front-end elif method == 'custom': if 'content' in data: self._handle_custom_msg(data['content']) def _handle_receive_state(self, sync_data): """Called when a state is received from the front-end.""" for name in self.keys: if name in sync_data: value = self._unpack_widgets(sync_data[name]) with self._lock_property(name, value): setattr(self, name, value) def _handle_custom_msg(self, content): """Called when a custom msg is received.""" self._msg_callbacks(self, content) def _handle_property_changed(self, name, old, new): """Called when a property has been changed.""" # Make sure this isn't information that the front-end just sent us. if self._should_send_property(name, new): # Send new state to front-end self.send_state(key=name) def _handle_displayed(self, **kwargs): """Called when a view has been displayed for this widget instance""" self._display_callbacks(self, **kwargs) def _pack_widgets(self, x): """Recursively converts all widget instances to model id strings. Children widgets will be stored and transmitted to the front-end by their model ids. Return value must be JSON-able.""" if isinstance(x, dict): return {k: self._pack_widgets(v) for k, v in x.items()} elif isinstance(x, (list, tuple)): return [self._pack_widgets(v) for v in x] elif isinstance(x, Widget): return x.model_id else: return x # Value must be JSON-able def _unpack_widgets(self, x): """Recursively converts all model id strings to widget instances. Children widgets will be stored and transmitted to the front-end by their model ids.""" if isinstance(x, dict): return {k: self._unpack_widgets(v) for k, v in x.items()} elif isinstance(x, (list, tuple)): return [self._unpack_widgets(v) for v in x] elif isinstance(x, string_types): return x if x not in Widget.widgets else Widget.widgets[x] else: return x def _ipython_display_(self, **kwargs): """Called when `IPython.display.display` is called on the widget.""" # Show view. By sending a display message, the comm is opened and the # initial state is sent. self._send({"method": "display"}) self._handle_displayed(**kwargs) def _send(self, msg): """Sends a message to the model in the front-end.""" self.comm.send(msg)
class KernelManager(LoggingConfigurable, ConnectionFileMixin): """Manages a single kernel in a subprocess on this host. This version starts kernels with Popen. """ # The PyZMQ Context to use for communication with the kernel. context = Instance(zmq.Context) def _context_default(self): return zmq.Context.instance() # The Session to use for communication with the kernel. session = Instance(Session) def _session_default(self): return Session(parent=self) # the class to create with our `client` method client_class = DottedObjectName( 'IPython.kernel.blocking.BlockingKernelClient') client_factory = Type() def _client_class_changed(self, name, old, new): self.client_factory = import_item(str(new)) # The kernel process with which the KernelManager is communicating. # generally a Popen instance kernel = Any() kernel_cmd = List(Unicode, config=True, help="""The Popen Command to launch the kernel. Override this if you have a custom """) def _kernel_cmd_changed(self, name, old, new): self.ipython_kernel = False ipython_kernel = Bool(True) # Protected traits _launch_args = Any() _control_socket = Any() _restarter = Any() autorestart = Bool(False, config=True, help="""Should we autorestart the kernel if it dies.""") def __del__(self): self._close_control_socket() self.cleanup_connection_file() #-------------------------------------------------------------------------- # Kernel restarter #-------------------------------------------------------------------------- def start_restarter(self): pass def stop_restarter(self): pass def add_restart_callback(self, callback, event='restart'): """register a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.add_callback(callback, event) def remove_restart_callback(self, callback, event='restart'): """unregister a callback to be called when a kernel is restarted""" if self._restarter is None: return self._restarter.remove_callback(callback, event) #-------------------------------------------------------------------------- # create a Client connected to our Kernel #-------------------------------------------------------------------------- def client(self, **kwargs): """Create a client configured to connect to our kernel""" if self.client_factory is None: self.client_factory = import_item(self.client_class) kw = {} kw.update(self.get_connection_info()) kw.update( dict( connection_file=self.connection_file, session=self.session, parent=self, )) # add kwargs last, for manual overrides kw.update(kwargs) return self.client_factory(**kw) #-------------------------------------------------------------------------- # Kernel management #-------------------------------------------------------------------------- def format_kernel_cmd(self, **kw): """replace templated args (e.g. {connection_file})""" if self.kernel_cmd: cmd = self.kernel_cmd else: cmd = make_ipkernel_cmd( 'from IPython.kernel.zmq.kernelapp import main; main()', **kw) ns = dict(connection_file=self.connection_file) ns.update(self._launch_args) pat = re.compile(r'\{([A-Za-z0-9_]+)\}') def from_ns(match): """Get the key out of ns if it's there, otherwise no change.""" return ns.get(match.group(1), match.group()) return [pat.sub(from_ns, arg) for arg in cmd] def _launch_kernel(self, kernel_cmd, **kw): """actually launch the kernel override in a subclass to launch kernel subprocesses differently """ return launch_kernel(kernel_cmd, **kw) # Control socket used for polite kernel shutdown def _connect_control_socket(self): if self._control_socket is None: self._control_socket = self.connect_control() self._control_socket.linger = 100 def _close_control_socket(self): if self._control_socket is None: return self._control_socket.close() self._control_socket = None def start_kernel(self, **kw): """Starts a kernel on this host in a separate process. If random ports (port=0) are being used, this method must be called before the channels are created. Parameters ---------- **kw : optional keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError( "Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " "Currently valid addresses are: %s" % local_ips()) # write connection file / get default ports self.write_connection_file() # save kwargs for use in restart self._launch_args = kw.copy() # build the Popen cmd kernel_cmd = self.format_kernel_cmd(**kw) # launch the kernel subprocess self.kernel = self._launch_kernel(kernel_cmd, ipython_kernel=self.ipython_kernel, **kw) self.start_restarter() self._connect_control_socket() def _send_shutdown_request(self, restart=False): """TODO: send a shutdown request via control channel""" content = dict(restart=restart) msg = self.session.msg("shutdown_request", content=content) self.session.send(self._control_socket, msg) def shutdown_kernel(self, now=False, restart=False): """Attempts to the stop the kernel process cleanly. This attempts to shutdown the kernels cleanly by: 1. Sending it a shutdown message over the shell channel. 2. If that fails, the kernel is shutdown forcibly by sending it a signal. Parameters ---------- now : bool Should the kernel be forcible killed *now*. This skips the first, nice shutdown attempt. restart: bool Will this kernel be restarted after it is shutdown. When this is True, connection files will not be cleaned up. """ # Stop monitoring for restarting while we shutdown. self.stop_restarter() # FIXME: Shutdown does not work on Windows due to ZMQ errors! if sys.platform == 'win32': self._kill_kernel() return if now: if self.has_kernel: self._kill_kernel() else: # Don't send any additional kernel kill messages immediately, to give # the kernel a chance to properly execute shutdown actions. Wait for at # most 1s, checking every 0.1s. self._send_shutdown_request(restart=restart) for i in range(10): if self.is_alive(): time.sleep(0.1) else: break else: # OK, we've waited long enough. if self.has_kernel: self._kill_kernel() if not restart: self.cleanup_connection_file() self.cleanup_ipc_files() else: self.cleanup_ipc_files() def restart_kernel(self, now=False, **kw): """Restarts a kernel with the arguments that were used to launch it. If the old kernel was launched with random ports, the same ports will be used for the new kernel. The same connection file is used again. Parameters ---------- now : bool, optional If True, the kernel is forcefully restarted *immediately*, without having a chance to do any cleanup action. Otherwise the kernel is given 1s to clean up before a forceful restart is issued. In all cases the kernel is restarted, the only difference is whether it is given a chance to perform a clean shutdown or not. **kw : optional Any options specified here will overwrite those used to launch the kernel. """ if self._launch_args is None: raise RuntimeError("Cannot restart the kernel. " "No previous call to 'start_kernel'.") else: # Stop currently running kernel. self.shutdown_kernel(now=now, restart=True) # Start new kernel. self._launch_args.update(kw) self.start_kernel(**self._launch_args) # FIXME: Messages get dropped in Windows due to probable ZMQ bug # unless there is some delay here. if sys.platform == 'win32': time.sleep(0.2) @property def has_kernel(self): """Has a kernel been started that we are managing.""" return self.kernel is not None def _kill_kernel(self): """Kill the running kernel. This is a private method, callers should use shutdown_kernel(now=True). """ if self.has_kernel: # Signal the kernel to terminate (sends SIGKILL on Unix and calls # TerminateProcess() on Win32). try: self.kernel.kill() except OSError as e: # In Windows, we will get an Access Denied error if the process # has already terminated. Ignore it. if sys.platform == 'win32': if e.winerror != 5: raise # On Unix, we may get an ESRCH error if the process has already # terminated. Ignore it. else: from errno import ESRCH if e.errno != ESRCH: raise # Block until the kernel terminates. self.kernel.wait() self.kernel = None else: raise RuntimeError("Cannot kill kernel. No kernel is running!") def interrupt_kernel(self): """Interrupts the kernel by sending it a signal. Unlike ``signal_kernel``, this operation is well supported on all platforms. """ if self.has_kernel: if sys.platform == 'win32': from .zmq.parentpoller import ParentPollerWindows as Poller Poller.send_interrupt(self.kernel.win32_interrupt_event) else: self.kernel.send_signal(signal.SIGINT) else: raise RuntimeError( "Cannot interrupt kernel. No kernel is running!") def signal_kernel(self, signum): """Sends a signal to the kernel. Note that since only SIGTERM is supported on Windows, this function is only useful on Unix systems. """ if self.has_kernel: self.kernel.send_signal(signum) else: raise RuntimeError("Cannot signal kernel. No kernel is running!") def is_alive(self): """Is the kernel process still running?""" if self.has_kernel: if self.kernel.poll() is None: return True else: return False else: # we don't have a kernel return False
class LenListTrait(HasTraits): value = List(Int, [0], minlen=1, maxlen=2)
class ListTrait(HasTraits): value = List(Int)
class Widget(LoggingConfigurable): #------------------------------------------------------------------------- # Class attributes #------------------------------------------------------------------------- _widget_construction_callback = None widgets = {} widget_types = {} @staticmethod def on_widget_constructed(callback): """Registers a callback to be called when a widget is constructed. The callback must have the following signature: callback(widget)""" Widget._widget_construction_callback = callback @staticmethod def _call_widget_constructed(widget): """Static method, called when a widget is constructed.""" if Widget._widget_construction_callback is not None and callable( Widget._widget_construction_callback): Widget._widget_construction_callback(widget) @staticmethod def handle_comm_opened(comm, msg): """Static method, called when a widget is constructed.""" widget_class = import_item(msg['content']['data']['widget_class']) widget = widget_class(comm=comm) #------------------------------------------------------------------------- # Traits #------------------------------------------------------------------------- _model_module = Unicode(None, allow_none=True, help="""A requirejs module name in which to find _model_name. If empty, look in the global registry.""" ) _model_name = Unicode('WidgetModel', help="""Name of the backbone model registered in the front-end to create and sync this widget with.""") _view_module = Unicode( help="""A requirejs module in which to find _view_name. If empty, look in the global registry.""", sync=True) _view_name = Unicode(None, allow_none=True, help="""Default view registered in the front-end to use to represent the widget.""", sync=True) comm = Instance('IPython.kernel.comm.Comm') msg_throttle = Int(3, sync=True, help="""Maximum number of msgs the front-end can send before receiving an idle msg from the back-end.""") version = Int(0, sync=True, help="""Widget's version""") keys = List() def _keys_default(self): return [name for name in self.traits(sync=True)] _property_lock = Tuple((None, None)) _send_state_lock = Int(0) _states_to_send = Set(allow_none=False) _display_callbacks = Instance(CallbackDispatcher, ()) _msg_callbacks = Instance(CallbackDispatcher, ()) #------------------------------------------------------------------------- # (Con/de)structor #------------------------------------------------------------------------- def __init__(self, **kwargs): """Public constructor""" self._model_id = kwargs.pop('model_id', None) super(Widget, self).__init__(**kwargs) Widget._call_widget_constructed(self) self.open() def __del__(self): """Object disposal""" self.close() #------------------------------------------------------------------------- # Properties #------------------------------------------------------------------------- def open(self): """Open a comm to the frontend if one isn't already open.""" if self.comm is None: args = dict(target_name='ipython.widget', data={ 'model_name': self._model_name, 'model_module': self._model_module }) if self._model_id is not None: args['comm_id'] = self._model_id self.comm = Comm(**args) def _comm_changed(self, name, new): """Called when the comm is changed.""" if new is None: return self._model_id = self.model_id self.comm.on_msg(self._handle_msg) Widget.widgets[self.model_id] = self # first update self.send_state() @property def model_id(self): """Gets the model id of this widget. If a Comm doesn't exist yet, a Comm will be created automagically.""" return self.comm.comm_id #------------------------------------------------------------------------- # Methods #------------------------------------------------------------------------- def close(self): """Close method. Closes the underlying comm. When the comm is closed, all of the widget views are automatically removed from the front-end.""" if self.comm is not None: Widget.widgets.pop(self.model_id, None) self.comm.close() self.comm = None def send_state(self, key=None): """Sends the widget state, or a piece of it, to the front-end. Parameters ---------- key : unicode, or iterable (optional) A single property's name or iterable of property names to sync with the front-end. """ self._send({"method": "update", "state": self.get_state(key=key)}) def get_state(self, key=None): """Gets the widget state, or a piece of it. Parameters ---------- key : unicode or iterable (optional) A single property's name or iterable of property names to get. """ if key is None: keys = self.keys elif isinstance(key, string_types): keys = [key] elif isinstance(key, collections.Iterable): keys = key else: raise ValueError( "key must be a string, an iterable of keys, or None") state = {} for k in keys: f = self.trait_metadata(k, 'to_json', self._trait_to_json) value = getattr(self, k) state[k] = f(value) return state def set_state(self, sync_data): """Called when a state is received from the front-end.""" for name in self.keys: if name in sync_data: json_value = sync_data[name] from_json = self.trait_metadata(name, 'from_json', self._trait_from_json) with self._lock_property(name, json_value): setattr(self, name, from_json(json_value)) def send(self, content): """Sends a custom msg to the widget model in the front-end. Parameters ---------- content : dict Content of the message to send. """ self._send({"method": "custom", "content": content}) def on_msg(self, callback, remove=False): """(Un)Register a custom msg receive callback. Parameters ---------- callback: callable callback will be passed two arguments when a message arrives:: callback(widget, content) remove: bool True if the callback should be unregistered.""" self._msg_callbacks.register_callback(callback, remove=remove) def on_displayed(self, callback, remove=False): """(Un)Register a widget displayed callback. Parameters ---------- callback: method handler Must have a signature of:: callback(widget, **kwargs) kwargs from display are passed through without modification. remove: bool True if the callback should be unregistered.""" self._display_callbacks.register_callback(callback, remove=remove) #------------------------------------------------------------------------- # Support methods #------------------------------------------------------------------------- @contextmanager def _lock_property(self, key, value): """Lock a property-value pair. The value should be the JSON state of the property. NOTE: This, in addition to the single lock for all state changes, is flawed. In the future we may want to look into buffering state changes back to the front-end.""" self._property_lock = (key, value) try: yield finally: self._property_lock = (None, None) @contextmanager def hold_sync(self): """Hold syncing any state until the context manager is released""" # We increment a value so that this can be nested. Syncing will happen when # all levels have been released. self._send_state_lock += 1 try: yield finally: self._send_state_lock -= 1 if self._send_state_lock == 0: self.send_state(self._states_to_send) self._states_to_send.clear() def _should_send_property(self, key, value): """Check the property lock (property_lock)""" to_json = self.trait_metadata(key, 'to_json', self._trait_to_json) if (key == self._property_lock[0] and to_json(value) == self._property_lock[1]): return False elif self._send_state_lock > 0: self._states_to_send.add(key) return False else: return True # Event handlers @_show_traceback def _handle_msg(self, msg): """Called when a msg is received from the front-end""" data = msg['content']['data'] method = data['method'] # Handle backbone sync methods CREATE, PATCH, and UPDATE all in one. if method == 'backbone': if 'sync_data' in data: sync_data = data['sync_data'] self.set_state(sync_data) # handles all methods # Handle a state request. elif method == 'request_state': self.send_state() # Handle a custom msg from the front-end. elif method == 'custom': if 'content' in data: self._handle_custom_msg(data['content']) # Catch remainder. else: self.log.error( 'Unknown front-end to back-end widget msg with method "%s"' % method) def _handle_custom_msg(self, content): """Called when a custom msg is received.""" self._msg_callbacks(self, content) def _notify_trait(self, name, old_value, new_value): """Called when a property has been changed.""" # Trigger default traitlet callback machinery. This allows any user # registered validation to be processed prior to allowing the widget # machinery to handle the state. LoggingConfigurable._notify_trait(self, name, old_value, new_value) # Send the state after the user registered callbacks for trait changes # have all fired (allows for user to validate values). if self.comm is not None and name in self.keys: # Make sure this isn't information that the front-end just sent us. if self._should_send_property(name, new_value): # Send new state to front-end self.send_state(key=name) def _handle_displayed(self, **kwargs): """Called when a view has been displayed for this widget instance""" self._display_callbacks(self, **kwargs) def _trait_to_json(self, x): """Convert a trait value to json Traverse lists/tuples and dicts and serialize their values as well. Replace any widgets with their model_id """ if isinstance(x, dict): return {k: self._trait_to_json(v) for k, v in x.items()} elif isinstance(x, (list, tuple)): return [self._trait_to_json(v) for v in x] elif isinstance(x, Widget): return "IPY_MODEL_" + x.model_id else: return x # Value must be JSON-able def _trait_from_json(self, x): """Convert json values to objects Replace any strings representing valid model id values to Widget references. """ if isinstance(x, dict): return {k: self._trait_from_json(v) for k, v in x.items()} elif isinstance(x, (list, tuple)): return [self._trait_from_json(v) for v in x] elif isinstance(x, string_types) and x.startswith( 'IPY_MODEL_') and x[10:] in Widget.widgets: # we want to support having child widgets at any level in a hierarchy # trusting that a widget UUID will not appear out in the wild return Widget.widgets[x[10:]] else: return x def _ipython_display_(self, **kwargs): """Called when `IPython.display.display` is called on the widget.""" # Show view. if self._view_name is not None: self._send({"method": "display"}) self._handle_displayed(**kwargs) def _send(self, msg): """Sends a message to the model in the front-end.""" self.comm.send(msg)
class TaskScheduler(SessionFactory): """Python TaskScheduler object. This is the simplest object that supports msg_id based DAG dependencies. *Only* task msg_ids are checked, not msg_ids of jobs submitted via the MUX queue. """ hwm = Integer(1, config=True, help="""specify the High Water Mark (HWM) for the downstream socket in the Task scheduler. This is the maximum number of allowed outstanding tasks on each engine. The default (1) means that only one task can be outstanding on each engine. Setting TaskScheduler.hwm=0 means there is no limit, and the engines continue to be assigned tasks while they are working, effectively hiding network latency behind computation, but can result in an imbalance of work when submitting many heterogenous tasks all at once. Any positive value greater than one is a compromise between the two. """) scheme_name = Enum( ('leastload', 'pure', 'lru', 'plainrandom', 'weighted', 'twobin'), 'leastload', config=True, allow_none=False, help="""select the task scheduler scheme [default: Python LRU] Options are: 'pure', 'lru', 'plainrandom', 'weighted', 'twobin','leastload'""" ) def _scheme_name_changed(self, old, new): self.log.debug("Using scheme %r" % new) self.scheme = globals()[new] # input arguments: scheme = Instance(FunctionType) # function for determining the destination def _scheme_default(self): return leastload client_stream = Instance(zmqstream.ZMQStream) # client-facing stream engine_stream = Instance(zmqstream.ZMQStream) # engine-facing stream notifier_stream = Instance(zmqstream.ZMQStream) # hub-facing sub stream mon_stream = Instance(zmqstream.ZMQStream) # hub-facing pub stream query_stream = Instance(zmqstream.ZMQStream) # hub-facing DEALER stream # internals: queue = Instance(deque) # sorted list of Jobs def _queue_default(self): return deque() queue_map = Dict() # dict by msg_id of Jobs (for O(1) access to the Queue) graph = Dict() # dict by msg_id of [ msg_ids that depend on key ] retries = Dict() # dict by msg_id of retries remaining (non-neg ints) # waiting = List() # list of msg_ids ready to run, but haven't due to HWM pending = Dict() # dict by engine_uuid of submitted tasks completed = Dict() # dict by engine_uuid of completed tasks failed = Dict() # dict by engine_uuid of failed tasks destinations = Dict( ) # dict by msg_id of engine_uuids where jobs ran (reverse of completed+failed) clients = Dict() # dict by msg_id for who submitted the task targets = List() # list of target IDENTs loads = List() # list of engine loads # full = Set() # set of IDENTs that have HWM outstanding tasks all_completed = Set() # set of all completed tasks all_failed = Set() # set of all failed tasks all_done = Set() # set of all finished tasks=union(completed,failed) all_ids = Set() # set of all submitted task IDs ident = CBytes() # ZMQ identity. This should just be self.session.session # but ensure Bytes def _ident_default(self): return self.session.bsession def start(self): self.query_stream.on_recv(self.dispatch_query_reply) self.session.send(self.query_stream, "connection_request", {}) self.engine_stream.on_recv(self.dispatch_result, copy=False) self.client_stream.on_recv(self.dispatch_submission, copy=False) self._notification_handlers = dict( registration_notification=self._register_engine, unregistration_notification=self._unregister_engine) self.notifier_stream.on_recv(self.dispatch_notification) self.log.info("Scheduler started [%s]" % self.scheme_name) def resume_receiving(self): """Resume accepting jobs.""" self.client_stream.on_recv(self.dispatch_submission, copy=False) def stop_receiving(self): """Stop accepting jobs while there are no engines. Leave them in the ZMQ queue.""" self.client_stream.on_recv(None) #----------------------------------------------------------------------- # [Un]Registration Handling #----------------------------------------------------------------------- def dispatch_query_reply(self, msg): """handle reply to our initial connection request""" try: idents, msg = self.session.feed_identities(msg) except ValueError: self.log.warn("task::Invalid Message: %r", msg) return try: msg = self.session.unserialize(msg) except ValueError: self.log.warn("task::Unauthorized message from: %r" % idents) return content = msg['content'] for uuid in content.get('engines', {}).values(): self._register_engine(cast_bytes(uuid)) @util.log_errors def dispatch_notification(self, msg): """dispatch register/unregister events.""" try: idents, msg = self.session.feed_identities(msg) except ValueError: self.log.warn("task::Invalid Message: %r", msg) return try: msg = self.session.unserialize(msg) except ValueError: self.log.warn("task::Unauthorized message from: %r" % idents) return msg_type = msg['header']['msg_type'] handler = self._notification_handlers.get(msg_type, None) if handler is None: self.log.error("Unhandled message type: %r" % msg_type) else: try: handler(cast_bytes(msg['content']['uuid'])) except Exception: self.log.error("task::Invalid notification msg: %r", msg, exc_info=True) def _register_engine(self, uid): """New engine with ident `uid` became available.""" # head of the line: self.targets.insert(0, uid) self.loads.insert(0, 0) # initialize sets self.completed[uid] = set() self.failed[uid] = set() self.pending[uid] = {} # rescan the graph: self.update_graph(None) def _unregister_engine(self, uid): """Existing engine with ident `uid` became unavailable.""" if len(self.targets) == 1: # this was our only engine pass # handle any potentially finished tasks: self.engine_stream.flush() # don't pop destinations, because they might be used later # map(self.destinations.pop, self.completed.pop(uid)) # map(self.destinations.pop, self.failed.pop(uid)) # prevent this engine from receiving work idx = self.targets.index(uid) self.targets.pop(idx) self.loads.pop(idx) # wait 5 seconds before cleaning up pending jobs, since the results might # still be incoming if self.pending[uid]: dc = ioloop.DelayedCallback( lambda: self.handle_stranded_tasks(uid), 5000, self.loop) dc.start() else: self.completed.pop(uid) self.failed.pop(uid) def handle_stranded_tasks(self, engine): """Deal with jobs resident in an engine that died.""" lost = self.pending[engine] for msg_id in lost.keys(): if msg_id not in self.pending[engine]: # prevent double-handling of messages continue raw_msg = lost[msg_id].raw_msg idents, msg = self.session.feed_identities(raw_msg, copy=False) parent = self.session.unpack(msg[1].bytes) idents = [engine, idents[0]] # build fake error reply try: raise error.EngineError( "Engine %r died while running task %r" % (engine, msg_id)) except: content = error.wrap_exception() # build fake metadata md = dict( status=u'error', engine=engine.decode('ascii'), date=datetime.now(), ) msg = self.session.msg('apply_reply', content, parent=parent, metadata=md) raw_reply = list( map(zmq.Message, self.session.serialize(msg, ident=idents))) # and dispatch it self.dispatch_result(raw_reply) # finally scrub completed/failed lists self.completed.pop(engine) self.failed.pop(engine) #----------------------------------------------------------------------- # Job Submission #----------------------------------------------------------------------- @util.log_errors def dispatch_submission(self, raw_msg): """Dispatch job submission to appropriate handlers.""" # ensure targets up to date: self.notifier_stream.flush() try: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unserialize(msg, content=False, copy=False) except Exception: self.log.error("task::Invaid task msg: %r" % raw_msg, exc_info=True) return # send to monitor self.mon_stream.send_multipart([b'intask'] + raw_msg, copy=False) header = msg['header'] md = msg['metadata'] msg_id = header['msg_id'] self.all_ids.add(msg_id) # get targets as a set of bytes objects # from a list of unicode objects targets = md.get('targets', []) targets = set(map(cast_bytes, targets)) retries = md.get('retries', 0) self.retries[msg_id] = retries # time dependencies after = md.get('after', None) if after: after = Dependency(after) if after.all: if after.success: after = Dependency( after.difference(self.all_completed), success=after.success, failure=after.failure, all=after.all, ) if after.failure: after = Dependency( after.difference(self.all_failed), success=after.success, failure=after.failure, all=after.all, ) if after.check(self.all_completed, self.all_failed): # recast as empty set, if `after` already met, # to prevent unnecessary set comparisons after = MET else: after = MET # location dependencies follow = Dependency(md.get('follow', [])) timeout = md.get('timeout', None) if timeout: timeout = float(timeout) job = Job( msg_id=msg_id, raw_msg=raw_msg, idents=idents, msg=msg, header=header, targets=targets, after=after, follow=follow, timeout=timeout, metadata=md, ) # validate and reduce dependencies: for dep in after, follow: if not dep: # empty dependency continue # check valid: if msg_id in dep or dep.difference(self.all_ids): self.queue_map[msg_id] = job return self.fail_unreachable(msg_id, error.InvalidDependency) # check if unreachable: if dep.unreachable(self.all_completed, self.all_failed): self.queue_map[msg_id] = job return self.fail_unreachable(msg_id) if after.check(self.all_completed, self.all_failed): # time deps already met, try to run if not self.maybe_run(job): # can't run yet if msg_id not in self.all_failed: # could have failed as unreachable self.save_unmet(job) else: self.save_unmet(job) def job_timeout(self, job, timeout_id): """callback for a job's timeout. The job may or may not have been run at this point. """ if job.timeout_id != timeout_id: # not the most recent call return now = time.time() if job.timeout >= (now + 1): self.log.warn("task %s timeout fired prematurely: %s > %s", job.msg_id, job.timeout, now) if job.msg_id in self.queue_map: # still waiting, but ran out of time self.log.info("task %r timed out", job.msg_id) self.fail_unreachable(job.msg_id, error.TaskTimeout) def fail_unreachable(self, msg_id, why=error.ImpossibleDependency): """a task has become unreachable, send a reply with an ImpossibleDependency error.""" if msg_id not in self.queue_map: self.log.error("task %r already failed!", msg_id) return job = self.queue_map.pop(msg_id) # lazy-delete from the queue job.removed = True for mid in job.dependents: if mid in self.graph: self.graph[mid].remove(msg_id) try: raise why() except: content = error.wrap_exception() self.log.debug("task %r failing as unreachable with: %s", msg_id, content['ename']) self.all_done.add(msg_id) self.all_failed.add(msg_id) msg = self.session.send(self.client_stream, 'apply_reply', content, parent=job.header, ident=job.idents) self.session.send(self.mon_stream, msg, ident=[b'outtask'] + job.idents) self.update_graph(msg_id, success=False) def available_engines(self): """return a list of available engine indices based on HWM""" if not self.hwm: return list(range(len(self.targets))) available = [] for idx in range(len(self.targets)): if self.loads[idx] < self.hwm: available.append(idx) return available def maybe_run(self, job): """check location dependencies, and run if they are met.""" msg_id = job.msg_id self.log.debug("Attempting to assign task %s", msg_id) available = self.available_engines() if not available: # no engines, definitely can't run return False if job.follow or job.targets or job.blacklist or self.hwm: # we need a can_run filter def can_run(idx): # check hwm if self.hwm and self.loads[idx] == self.hwm: return False target = self.targets[idx] # check blacklist if target in job.blacklist: return False # check targets if job.targets and target not in job.targets: return False # check follow return job.follow.check(self.completed[target], self.failed[target]) indices = list(filter(can_run, available)) if not indices: # couldn't run if job.follow.all: # check follow for impossibility dests = set() relevant = set() if job.follow.success: relevant = self.all_completed if job.follow.failure: relevant = relevant.union(self.all_failed) for m in job.follow.intersection(relevant): dests.add(self.destinations[m]) if len(dests) > 1: self.queue_map[msg_id] = job self.fail_unreachable(msg_id) return False if job.targets: # check blacklist+targets for impossibility job.targets.difference_update(job.blacklist) if not job.targets or not job.targets.intersection( self.targets): self.queue_map[msg_id] = job self.fail_unreachable(msg_id) return False return False else: indices = None self.submit_task(job, indices) return True def save_unmet(self, job): """Save a message for later submission when its dependencies are met.""" msg_id = job.msg_id self.log.debug("Adding task %s to the queue", msg_id) self.queue_map[msg_id] = job self.queue.append(job) # track the ids in follow or after, but not those already finished for dep_id in job.after.union(job.follow).difference(self.all_done): if dep_id not in self.graph: self.graph[dep_id] = set() self.graph[dep_id].add(msg_id) # schedule timeout callback if job.timeout: timeout_id = job.timeout_id = job.timeout_id + 1 self.loop.add_timeout(time.time() + job.timeout, lambda: self.job_timeout(job, timeout_id)) def submit_task(self, job, indices=None): """Submit a task to any of a subset of our targets.""" if indices: loads = [self.loads[i] for i in indices] else: loads = self.loads idx = self.scheme(loads) if indices: idx = indices[idx] target = self.targets[idx] # print (target, map(str, msg[:3])) # send job to the engine self.engine_stream.send(target, flags=zmq.SNDMORE, copy=False) self.engine_stream.send_multipart(job.raw_msg, copy=False) # update load self.add_job(idx) self.pending[target][job.msg_id] = job # notify Hub content = dict(msg_id=job.msg_id, engine_id=target.decode('ascii')) self.session.send(self.mon_stream, 'task_destination', content=content, ident=[b'tracktask', self.ident]) #----------------------------------------------------------------------- # Result Handling #----------------------------------------------------------------------- @util.log_errors def dispatch_result(self, raw_msg): """dispatch method for result replies""" try: idents, msg = self.session.feed_identities(raw_msg, copy=False) msg = self.session.unserialize(msg, content=False, copy=False) engine = idents[0] try: idx = self.targets.index(engine) except ValueError: pass # skip load-update for dead engines else: self.finish_job(idx) except Exception: self.log.error("task::Invalid result: %r", raw_msg, exc_info=True) return md = msg['metadata'] parent = msg['parent_header'] if md.get('dependencies_met', True): success = (md['status'] == 'ok') msg_id = parent['msg_id'] retries = self.retries[msg_id] if not success and retries > 0: # failed self.retries[msg_id] = retries - 1 self.handle_unmet_dependency(idents, parent) else: del self.retries[msg_id] # relay to client and update graph self.handle_result(idents, parent, raw_msg, success) # send to Hub monitor self.mon_stream.send_multipart([b'outtask'] + raw_msg, copy=False) else: self.handle_unmet_dependency(idents, parent) def handle_result(self, idents, parent, raw_msg, success=True): """handle a real task result, either success or failure""" # first, relay result to client engine = idents[0] client = idents[1] # swap_ids for ROUTER-ROUTER mirror raw_msg[:2] = [client, engine] # print (map(str, raw_msg[:4])) self.client_stream.send_multipart(raw_msg, copy=False) # now, update our data structures msg_id = parent['msg_id'] self.pending[engine].pop(msg_id) if success: self.completed[engine].add(msg_id) self.all_completed.add(msg_id) else: self.failed[engine].add(msg_id) self.all_failed.add(msg_id) self.all_done.add(msg_id) self.destinations[msg_id] = engine self.update_graph(msg_id, success) def handle_unmet_dependency(self, idents, parent): """handle an unmet dependency""" engine = idents[0] msg_id = parent['msg_id'] job = self.pending[engine].pop(msg_id) job.blacklist.add(engine) if job.blacklist == job.targets: self.queue_map[msg_id] = job self.fail_unreachable(msg_id) elif not self.maybe_run(job): # resubmit failed if msg_id not in self.all_failed: # put it back in our dependency tree self.save_unmet(job) if self.hwm: try: idx = self.targets.index(engine) except ValueError: pass # skip load-update for dead engines else: if self.loads[idx] == self.hwm - 1: self.update_graph(None) def update_graph(self, dep_id=None, success=True): """dep_id just finished. Update our dependency graph and submit any jobs that just became runnable. Called with dep_id=None to update entire graph for hwm, but without finishing a task. """ # print ("\n\n***********") # pprint (dep_id) # pprint (self.graph) # pprint (self.queue_map) # pprint (self.all_completed) # pprint (self.all_failed) # print ("\n\n***********\n\n") # update any jobs that depended on the dependency msg_ids = self.graph.pop(dep_id, []) # recheck *all* jobs if # a) we have HWM and an engine just become no longer full # or b) dep_id was given as None if dep_id is None or self.hwm and any( [load == self.hwm - 1 for load in self.loads]): jobs = self.queue using_queue = True else: using_queue = False jobs = deque(sorted(self.queue_map[msg_id] for msg_id in msg_ids)) to_restore = [] while jobs: job = jobs.popleft() if job.removed: continue msg_id = job.msg_id put_it_back = True if job.after.unreachable(self.all_completed, self.all_failed)\ or job.follow.unreachable(self.all_completed, self.all_failed): self.fail_unreachable(msg_id) put_it_back = False elif job.after.check(self.all_completed, self.all_failed): # time deps met, maybe run if self.maybe_run(job): put_it_back = False self.queue_map.pop(msg_id) for mid in job.dependents: if mid in self.graph: self.graph[mid].remove(msg_id) # abort the loop if we just filled up all of our engines. # avoids an O(N) operation in situation of full queue, # where graph update is triggered as soon as an engine becomes # non-full, and all tasks after the first are checked, # even though they can't run. if not self.available_engines(): break if using_queue and put_it_back: # popped a job from the queue but it neither ran nor failed, # so we need to put it back when we are done # make sure to_restore preserves the same ordering to_restore.append(job) # put back any tasks we popped but didn't run if using_queue: self.queue.extendleft(to_restore) #---------------------------------------------------------------------- # methods to be overridden by subclasses #---------------------------------------------------------------------- def add_job(self, idx): """Called after self.targets[idx] just got the job with header. Override with subclasses. The default ordering is simple LRU. The default loads are the number of outstanding jobs.""" self.loads[idx] += 1 for lis in (self.targets, self.loads): lis.append(lis.pop(idx)) def finish_job(self, idx): """Called after self.targets[idx] just finished a job. Override with subclasses.""" self.loads[idx] -= 1
class Exporter(LoggingConfigurable): """ Class containing methods that sequentially run a list of preprocessors on a NotebookNode object and then return the modified NotebookNode object and accompanying resources dict. """ file_extension = Unicode( 'txt', config=True, help="Extension of the file that should be written to disk" ) # MIME type of the result file, for HTTP response headers. # This is *not* a traitlet, because we want to be able to access it from # the class, not just on instances. output_mimetype = '' #Configurability, allows the user to easily add filters and preprocessors. preprocessors = List(config=True, help="""List of preprocessors, by name or namespace, to enable.""") _preprocessors = List() default_preprocessors = List(['IPython.nbconvert.preprocessors.coalesce_streams', 'IPython.nbconvert.preprocessors.SVG2PDFPreprocessor', 'IPython.nbconvert.preprocessors.ExtractOutputPreprocessor', 'IPython.nbconvert.preprocessors.CSSHTMLHeaderPreprocessor', 'IPython.nbconvert.preprocessors.RevealHelpPreprocessor', 'IPython.nbconvert.preprocessors.LatexPreprocessor', 'IPython.nbconvert.preprocessors.ClearOutputPreprocessor', 'IPython.nbconvert.preprocessors.ExecutePreprocessor', 'IPython.nbconvert.preprocessors.HighlightMagicsPreprocessor'], config=True, help="""List of preprocessors available by default, by name, namespace, instance, or type.""") def __init__(self, config=None, **kw): """ Public constructor Parameters ---------- config : config User configuration instance. """ with_default_config = self.default_config if config: with_default_config.merge(config) super(Exporter, self).__init__(config=with_default_config, **kw) self._init_preprocessors() @property def default_config(self): return Config() def from_notebook_node(self, nb, resources=None, **kw): """ Convert a notebook from a notebook node instance. Parameters ---------- nb : :class:`~IPython.nbformat.current.NotebookNode` Notebook node resources : dict Additional resources that can be accessed read/write by preprocessors and filters. **kw Ignored (?) """ nb_copy = copy.deepcopy(nb) resources = self._init_resources(resources) if 'language' in nb['metadata']: resources['language'] = nb['metadata']['language'].lower() # Preprocess nb_copy, resources = self._preprocess(nb_copy, resources) return nb_copy, resources def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- filename : str Full filename of the notebook file to open and convert. """ # Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': resources['metadata'] = ResourcesDict() basename = os.path.basename(filename) notebook_name = basename[:basename.rfind('.')] resources['metadata']['name'] = notebook_name modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) resources['metadata']['modified_date'] = modified_date.strftime(text.date_format) with io.open(filename, encoding='utf-8') as f: return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw) def from_file(self, file_stream, resources=None, **kw): """ Convert a notebook from a notebook file. Parameters ---------- file_stream : file-like object Notebook file-like object to convert. """ return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw) def register_preprocessor(self, preprocessor, enabled=False): """ Register a preprocessor. Preprocessors are classes that act upon the notebook before it is passed into the Jinja templating engine. preprocessors are also capable of passing additional information to the Jinja templating engine. Parameters ---------- preprocessor : preprocessor """ if preprocessor is None: raise TypeError('preprocessor') isclass = isinstance(preprocessor, type) constructed = not isclass # Handle preprocessor's registration based on it's type if constructed and isinstance(preprocessor, py3compat.string_types): # Preprocessor is a string, import the namespace and recursively call # this register_preprocessor method preprocessor_cls = import_item(preprocessor) return self.register_preprocessor(preprocessor_cls, enabled) if constructed and hasattr(preprocessor, '__call__'): # Preprocessor is a function, no need to construct it. # Register and return the preprocessor. if enabled: preprocessor.enabled = True self._preprocessors.append(preprocessor) return preprocessor elif isclass and isinstance(preprocessor, MetaHasTraits): # Preprocessor is configurable. Make sure to pass in new default for # the enabled flag if one was specified. self.register_preprocessor(preprocessor(parent=self), enabled) elif isclass: # Preprocessor is not configurable, construct it self.register_preprocessor(preprocessor(), enabled) else: # Preprocessor is an instance of something without a __call__ # attribute. raise TypeError('preprocessor') def _init_preprocessors(self): """ Register all of the preprocessors needed for this exporter, disabled unless specified explicitly. """ self._preprocessors = [] # Load default preprocessors (not necessarly enabled by default). for preprocessor in self.default_preprocessors: self.register_preprocessor(preprocessor) # Load user-specified preprocessors. Enable by default. for preprocessor in self.preprocessors: self.register_preprocessor(preprocessor, enabled=True) def _init_resources(self, resources): #Make sure the resources dict is of ResourcesDict type. if resources is None: resources = ResourcesDict() if not isinstance(resources, ResourcesDict): new_resources = ResourcesDict() new_resources.update(resources) resources = new_resources #Make sure the metadata extension exists in resources if 'metadata' in resources: if not isinstance(resources['metadata'], ResourcesDict): resources['metadata'] = ResourcesDict(resources['metadata']) else: resources['metadata'] = ResourcesDict() if not resources['metadata']['name']: resources['metadata']['name'] = 'Notebook' #Set the output extension resources['output_extension'] = self.file_extension return resources def _preprocess(self, nb, resources): """ Preprocess the notebook before passing it into the Jinja engine. To preprocess the notebook is to apply all of the Parameters ---------- nb : notebook node notebook that is being exported. resources : a dict of additional resources that can be accessed read/write by preprocessors """ # Do a copy.deepcopy first, # we are never safe enough with what the preprocessors could do. nbc = copy.deepcopy(nb) resc = copy.deepcopy(resources) #Run each preprocessor on the notebook. Carry the output along #to each preprocessor for preprocessor in self._preprocessors: nbc, resc = preprocessor(nbc, resc) return nbc, resc
class View(HasTraits): """Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes. Don't use this class, use subclasses. Methods ------- spin flushes incoming results and registration state changes control methods spin, and requesting `ids` also ensures up to date wait wait on one or more msg_ids execution methods apply legacy: execute, run data movement push, pull, scatter, gather query methods get_result, queue_status, purge_results, result_status control methods abort, shutdown """ # flags block = Bool(False) track = Bool(True) targets = Any() history = List() outstanding = Set() results = Dict() client = Instance('IPython.parallel.Client') _socket = Instance('zmq.Socket') _flag_names = List(['targets', 'block', 'track']) _targets = Any() _idents = Any() def __init__(self, client=None, socket=None, **flags): super(View, self).__init__(client=client, _socket=socket) self.block = client.block self.set_flags(**flags) assert not self.__class__ is View, "Don't use base View objects, use subclasses" def __repr__(self): strtargets = str(self.targets) if len(strtargets) > 16: strtargets = strtargets[:12] + '...]' return "<%s %s>" % (self.__class__.__name__, strtargets) def __len__(self): if isinstance(self.targets, list): return len(self.targets) elif isinstance(self.targets, int): return 1 else: return len(self.client) def set_flags(self, **kwargs): """set my attribute flags by keyword. Views determine behavior with a few attributes (`block`, `track`, etc.). These attributes can be set all at once by name with this method. Parameters ---------- block : bool whether to wait for results track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. """ for name, value in kwargs.iteritems(): if name not in self._flag_names: raise KeyError("Invalid name: %r" % name) else: setattr(self, name, value) @contextmanager def temp_flags(self, **kwargs): """temporarily set flags, for use in `with` statements. See set_flags for permanent setting of flags Examples -------- >>> view.track=False ... >>> with view.temp_flags(track=True): ... ar = view.apply(dostuff, my_big_array) ... ar.tracker.wait() # wait for send to finish >>> view.track False """ # preflight: save flags, and set temporaries saved_flags = {} for f in self._flag_names: saved_flags[f] = getattr(self, f) self.set_flags(**kwargs) # yield to the with-statement block try: yield finally: # postflight: restore saved flags self.set_flags(**saved_flags) #---------------------------------------------------------------- # apply #---------------------------------------------------------------- @sync_results @save_ids def _really_apply(self, f, args, kwargs, block=None, **options): """wrapper for client.send_apply_request""" raise NotImplementedError("Implement in subclasses") def apply(self, f, *args, **kwargs): """calls f(*args, **kwargs) on remote engines, returning the result. This method sets all apply flags via this View's attributes. if self.block is False: returns AsyncResult else: returns actual result of f(*args, **kwargs) """ return self._really_apply(f, args, kwargs) def apply_async(self, f, *args, **kwargs): """calls f(*args, **kwargs) on remote engines in a nonblocking manner. returns AsyncResult """ return self._really_apply(f, args, kwargs, block=False) @spin_after def apply_sync(self, f, *args, **kwargs): """calls f(*args, **kwargs) on remote engines in a blocking manner, returning the result. returns: actual result of f(*args, **kwargs) """ return self._really_apply(f, args, kwargs, block=True) #---------------------------------------------------------------- # wrappers for client and control methods #---------------------------------------------------------------- @sync_results def spin(self): """spin the client, and sync""" self.client.spin() @sync_results def wait(self, jobs=None, timeout=-1): """waits on one or more `jobs`, for up to `timeout` seconds. Parameters ---------- jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects ints are indices to self.history strs are msg_ids default: wait on all outstanding messages timeout : float a time in seconds, after which to give up. default is -1, which means no timeout Returns ------- True : when all msg_ids are done False : timeout reached, some msg_ids still outstanding """ if jobs is None: jobs = self.history return self.client.wait(jobs, timeout) def abort(self, jobs=None, targets=None, block=None): """Abort jobs on my engines. Parameters ---------- jobs : None, str, list of strs, optional if None: abort all jobs. else: abort specific msg_id(s). """ block = block if block is not None else self.block targets = targets if targets is not None else self.targets jobs = jobs if jobs is not None else list(self.outstanding) return self.client.abort(jobs=jobs, targets=targets, block=block) def queue_status(self, targets=None, verbose=False): """Fetch the Queue status of my engines""" targets = targets if targets is not None else self.targets return self.client.queue_status(targets=targets, verbose=verbose) def purge_results(self, jobs=[], targets=[]): """Instruct the controller to forget specific results.""" if targets is None or targets == 'all': targets = self.targets return self.client.purge_results(jobs=jobs, targets=targets) def shutdown(self, targets=None, restart=False, hub=False, block=None): """Terminates one or more engine processes, optionally including the hub. """ block = self.block if block is None else block if targets is None or targets == 'all': targets = self.targets return self.client.shutdown(targets=targets, restart=restart, hub=hub, block=block) @spin_after def get_result(self, indices_or_msg_ids=None): """return one or more results, specified by history index or msg_id. See client.get_result for details. """ if indices_or_msg_ids is None: indices_or_msg_ids = -1 if isinstance(indices_or_msg_ids, int): indices_or_msg_ids = self.history[indices_or_msg_ids] elif isinstance(indices_or_msg_ids, (list, tuple, set)): indices_or_msg_ids = list(indices_or_msg_ids) for i, index in enumerate(indices_or_msg_ids): if isinstance(index, int): indices_or_msg_ids[i] = self.history[index] return self.client.get_result(indices_or_msg_ids) #------------------------------------------------------------------- # Map #------------------------------------------------------------------- def map(self, f, *sequences, **kwargs): """override in subclasses""" raise NotImplementedError def map_async(self, f, *sequences, **kwargs): """Parallel version of builtin `map`, using this view's engines. This is equivalent to map(...block=False) See `self.map` for details. """ if 'block' in kwargs: raise TypeError( "map_async doesn't take a `block` keyword argument.") kwargs['block'] = False return self.map(f, *sequences, **kwargs) def map_sync(self, f, *sequences, **kwargs): """Parallel version of builtin `map`, using this view's engines. This is equivalent to map(...block=True) See `self.map` for details. """ if 'block' in kwargs: raise TypeError( "map_sync doesn't take a `block` keyword argument.") kwargs['block'] = True return self.map(f, *sequences, **kwargs) def imap(self, f, *sequences, **kwargs): """Parallel version of `itertools.imap`. See `self.map` for details. """ return iter(self.map_async(f, *sequences, **kwargs)) #------------------------------------------------------------------- # Decorators #------------------------------------------------------------------- def remote(self, block=True, **flags): """Decorator for making a RemoteFunction""" block = self.block if block is None else block return remote(self, block=block, **flags) def parallel(self, dist='b', block=None, **flags): """Decorator for making a ParallelFunction""" block = self.block if block is None else block return parallel(self, dist=dist, block=block, **flags)
class LazyConfigValue(HasTraits): """Proxy object for exposing methods on configurable containers Exposes: - append, extend, insert on lists - update on dicts - update, add on sets """ _value = None # list methods _extend = List() _prepend = List() def append(self, obj): self._extend.append(obj) def extend(self, other): self._extend.extend(other) def prepend(self, other): """like list.extend, but for the front""" self._prepend[:0] = other _inserts = List() def insert(self, index, other): if not isinstance(index, int): raise TypeError("An integer is required") self._inserts.append((index, other)) # dict methods # update is used for both dict and set _update = Any() def update(self, other): if self._update is None: if isinstance(other, dict): self._update = {} else: self._update = set() self._update.update(other) # set methods def add(self, obj): self.update({obj}) def get_value(self, initial): """construct the value from the initial one after applying any insert / extend / update changes """ if self._value is not None: return self._value value = copy.deepcopy(initial) if isinstance(value, list): for idx, obj in self._inserts: value.insert(idx, obj) value[:0] = self._prepend value.extend(self._extend) elif isinstance(value, dict): if self._update: value.update(self._update) elif isinstance(value, set): if self._update: value.update(self._update) self._value = value return value def to_dict(self): """return JSONable dict form of my data Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples. """ d = {} if self._update: d['update'] = self._update if self._extend: d['extend'] = self._extend if self._prepend: d['prepend'] = self._prepend elif self._inserts: d['inserts'] = self._inserts return d
class LoadBalancedView(View): """An load-balancing View that only executes via the Task scheduler. Load-balanced views can be created with the client's `view` method: >>> v = client.load_balanced_view() or targets can be specified, to restrict the potential destinations: >>> v = client.client.load_balanced_view([1,3]) which would restrict loadbalancing to between engines 1 and 3. """ follow = Any() after = Any() timeout = CFloat() retries = Integer(0) _task_scheme = Any() _flag_names = List( ['targets', 'block', 'track', 'follow', 'after', 'timeout', 'retries']) def __init__(self, client=None, socket=None, **flags): super(LoadBalancedView, self).__init__(client=client, socket=socket, **flags) self._task_scheme = client._task_scheme def _validate_dependency(self, dep): """validate a dependency. For use in `set_flags`. """ if dep is None or isinstance(dep, (basestring, AsyncResult, Dependency)): return True elif isinstance(dep, (list, set, tuple)): for d in dep: if not isinstance(d, (basestring, AsyncResult)): return False elif isinstance(dep, dict): if set(dep.keys()) != set(Dependency().as_dict().keys()): return False if not isinstance(dep['msg_ids'], list): return False for d in dep['msg_ids']: if not isinstance(d, basestring): return False else: return False return True def _render_dependency(self, dep): """helper for building jsonable dependencies from various input forms.""" if isinstance(dep, Dependency): return dep.as_dict() elif isinstance(dep, AsyncResult): return dep.msg_ids elif dep is None: return [] else: # pass to Dependency constructor return list(Dependency(dep)) def set_flags(self, **kwargs): """set my attribute flags by keyword. A View is a wrapper for the Client's apply method, but with attributes that specify keyword arguments, those attributes can be set by keyword argument with this method. Parameters ---------- block : bool whether to wait for results track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. after : Dependency or collection of msg_ids Only for load-balanced execution (targets=None) Specify a list of msg_ids as a time-based dependency. This job will only be run *after* the dependencies have been met. follow : Dependency or collection of msg_ids Only for load-balanced execution (targets=None) Specify a list of msg_ids as a location-based dependency. This job will only be run on an engine where this dependency is met. timeout : float/int or None Only for load-balanced execution (targets=None) Specify an amount of time (in seconds) for the scheduler to wait for dependencies to be met before failing with a DependencyTimeout. retries : int Number of times a task will be retried on failure. """ super(LoadBalancedView, self).set_flags(**kwargs) for name in ('follow', 'after'): if name in kwargs: value = kwargs[name] if self._validate_dependency(value): setattr(self, name, value) else: raise ValueError("Invalid dependency: %r" % value) if 'timeout' in kwargs: t = kwargs['timeout'] if not isinstance(t, (int, long, float, type(None))): raise TypeError("Invalid type for timeout: %r" % type(t)) if t is not None: if t < 0: raise ValueError("Invalid timeout: %s" % t) self.timeout = t @sync_results @save_ids def _really_apply(self, f, args=None, kwargs=None, block=None, track=None, after=None, follow=None, timeout=None, targets=None, retries=None): """calls f(*args, **kwargs) on a remote engine, returning the result. This method temporarily sets all of `apply`'s flags for a single call. Parameters ---------- f : callable args : list [default: empty] kwargs : dict [default: empty] block : bool [default: self.block] whether to block track : bool [default: self.track] whether to ask zmq to track the message, for safe non-copying sends !!!!!! TODO: THE REST HERE !!!! Returns ------- if self.block is False: returns AsyncResult else: returns actual result of f(*args, **kwargs) on the engine(s) This will be a list of self.targets is also a list (even length 1), or the single result if self.targets is an integer engine id """ # validate whether we can run if self._socket.closed: msg = "Task farming is disabled" if self._task_scheme == 'pure': msg += " because the pure ZMQ scheduler cannot handle" msg += " disappearing engines." raise RuntimeError(msg) if self._task_scheme == 'pure': # pure zmq scheme doesn't support extra features msg = "Pure ZMQ scheduler doesn't support the following flags:" "follow, after, retries, targets, timeout" if (follow or after or retries or targets or timeout): # hard fail on Scheduler flags raise RuntimeError(msg) if isinstance(f, dependent): # soft warn on functional dependencies warnings.warn(msg, RuntimeWarning) # build args args = [] if args is None else args kwargs = {} if kwargs is None else kwargs block = self.block if block is None else block track = self.track if track is None else track after = self.after if after is None else after retries = self.retries if retries is None else retries follow = self.follow if follow is None else follow timeout = self.timeout if timeout is None else timeout targets = self.targets if targets is None else targets if not isinstance(retries, int): raise TypeError('retries must be int, not %r' % type(retries)) if targets is None: idents = [] else: idents = self.client._build_targets(targets)[0] # ensure *not* bytes idents = [ident.decode() for ident in idents] after = self._render_dependency(after) follow = self._render_dependency(follow) subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries) msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track, subheader=subheader) tracker = None if track is False else msg['tracker'] ar = AsyncResult(self.client, msg['header']['msg_id'], fname=getname(f), targets=None, tracker=tracker) if block: try: return ar.get() except KeyboardInterrupt: pass return ar @spin_after @save_ids def map(self, f, *sequences, **kwargs): """view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult Parallel version of builtin `map`, load-balanced by this View. `block`, and `chunksize` can be specified by keyword only. Each `chunksize` elements will be a separate task, and will be load-balanced. This lets individual elements be available for iteration as soon as they arrive. Parameters ---------- f : callable function to be mapped *sequences: one or more sequences of matching length the sequences to be distributed and passed to `f` block : bool [default self.block] whether to wait for the result or not track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. chunksize : int [default 1] how many elements should be in each task. ordered : bool [default True] Whether the results should be gathered as they arrive, or enforce the order of submission. Only applies when iterating through AsyncMapResult as results arrive. Has no effect when block=True. Returns ------- if block=False: AsyncMapResult An object like AsyncResult, but which reassembles the sequence of results into a single list. AsyncMapResults can be iterated through before all results are complete. else: the result of map(f,*sequences) """ # default block = kwargs.get('block', self.block) chunksize = kwargs.get('chunksize', 1) ordered = kwargs.get('ordered', True) keyset = set(kwargs.keys()) extra_keys = keyset.difference_update(set(['block', 'chunksize'])) if extra_keys: raise TypeError("Invalid kwargs: %s" % list(extra_keys)) assert len(sequences) > 0, "must have some sequences to map onto!" pf = ParallelFunction(self, f, block=block, chunksize=chunksize, ordered=ordered) return pf.map(*sequences)
class LocalProcessLauncher(BaseLauncher): """Start and stop an external process in an asynchronous manner. This will launch the external process with a working directory of ``self.work_dir``. """ # This is used to to construct self.args, which is passed to # spawnProcess. cmd_and_args = List([]) poll_frequency = Integer(100) # in ms def __init__(self, work_dir=u'.', config=None, **kwargs): super(LocalProcessLauncher, self).__init__(work_dir=work_dir, config=config, **kwargs) self.process = None self.poller = None def find_args(self): return self.cmd_and_args def start(self): self.log.debug("Starting %s: %r", self.__class__.__name__, self.args) if self.state == 'before': self.process = Popen(self.args, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=os.environ, cwd=self.work_dir) if WINDOWS: self.stdout = forward_read_events(self.process.stdout) self.stderr = forward_read_events(self.process.stderr) else: self.stdout = self.process.stdout.fileno() self.stderr = self.process.stderr.fileno() self.loop.add_handler(self.stdout, self.handle_stdout, self.loop.READ) self.loop.add_handler(self.stderr, self.handle_stderr, self.loop.READ) self.poller = ioloop.PeriodicCallback(self.poll, self.poll_frequency, self.loop) self.poller.start() self.notify_start(self.process.pid) else: s = 'The process was already started and has state: %r' % self.state raise ProcessStateError(s) def stop(self): return self.interrupt_then_kill() def signal(self, sig): if self.state == 'running': if WINDOWS and sig != SIGINT: # use Windows tree-kill for better child cleanup check_output( ['taskkill', '-pid', str(self.process.pid), '-t', '-f']) else: self.process.send_signal(sig) def interrupt_then_kill(self, delay=2.0): """Send INT, wait a delay and then send KILL.""" try: self.signal(SIGINT) except Exception: self.log.debug("interrupt failed") pass self.killer = ioloop.DelayedCallback(lambda: self.signal(SIGKILL), delay * 1000, self.loop) self.killer.start() # callbacks, etc: def handle_stdout(self, fd, events): if WINDOWS: line = self.stdout.recv() else: line = self.process.stdout.readline() # a stopped process will be readable but return empty strings if line: self.log.debug(line[:-1]) else: self.poll() def handle_stderr(self, fd, events): if WINDOWS: line = self.stderr.recv() else: line = self.process.stderr.readline() # a stopped process will be readable but return empty strings if line: self.log.debug(line[:-1]) else: self.poll() def poll(self): status = self.process.poll() if status is not None: self.poller.stop() self.loop.remove_handler(self.stdout) self.loop.remove_handler(self.stderr) self.notify_stop(dict(exit_code=status, pid=self.process.pid)) return status
class IPythonConsoleApp(ConnectionFileMixin): name = 'ipython-console-mixin' description = """ The IPython Mixin Console. This class contains the common portions of console client (QtConsole, ZMQ-based terminal console, etc). It is not a full console, in that launched terminal subprocesses will not be able to accept input. The Console using this mixing supports various extra features beyond the single-process Terminal IPython shell, such as connecting to existing kernel, via: ipython <appname> --existing as well as tunnel via SSH """ classes = classes flags = Dict(flags) aliases = Dict(aliases) kernel_manager_class = KernelManager kernel_client_class = BlockingKernelClient kernel_argv = List(Unicode) # frontend flags&aliases to be stripped when building kernel_argv frontend_flags = Any(app_flags) frontend_aliases = Any(app_aliases) # create requested profiles by default, if they don't exist: auto_create = CBool(True) # connection info: sshserver = Unicode( '', config=True, help="""The SSH server to use to connect to the kernel.""") sshkey = Unicode( '', config=True, help="""Path to the ssh key to use for logging in to the ssh server.""" ) def _connection_file_default(self): return 'kernel-%i.json' % os.getpid() existing = CUnicode('', config=True, help="""Connect to an already running kernel""") kernel_name = Unicode('python', config=True, help="""The name of the default kernel to start.""") confirm_exit = CBool( True, config=True, help=""" Set to display confirmation dialog on exit. You can always use 'exit' or 'quit', to force a direct exit without any confirmation.""", ) def build_kernel_argv(self, argv=None): """build argv to be passed to kernel subprocess Override in subclasses if any args should be passed to the kernel """ self.kernel_argv = self.extra_args def init_connection_file(self): """find the connection file, and load the info if found. The current working directory and the current profile's security directory will be searched for the file if it is not given by absolute path. When attempting to connect to an existing kernel and the `--existing` argument does not match an existing file, it will be interpreted as a fileglob, and the matching file in the current profile's security dir with the latest access time will be used. After this method is called, self.connection_file contains the *full path* to the connection file, never just its name. """ if self.existing: try: cf = find_connection_file(self.existing) except Exception: self.log.critical( "Could not find existing kernel connection file %s", self.existing) self.exit(1) self.log.debug("Connecting to existing kernel: %s" % cf) self.connection_file = cf else: # not existing, check if we are going to write the file # and ensure that self.connection_file is a full path, not just the shortname try: cf = find_connection_file(self.connection_file) except Exception: # file might not exist if self.connection_file == os.path.basename( self.connection_file): # just shortname, put it in security dir cf = os.path.join(self.profile_dir.security_dir, self.connection_file) else: cf = self.connection_file self.connection_file = cf try: self.connection_file = filefind( self.connection_file, ['.', self.profile_dir.security_dir]) except IOError: self.log.debug("Connection File not found: %s", self.connection_file) return # should load_connection_file only be used for existing? # as it is now, this allows reusing ports if an existing # file is requested try: self.load_connection_file() except Exception: self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True) self.exit(1) def init_ssh(self): """set up ssh tunnels, if needed.""" if not self.existing or (not self.sshserver and not self.sshkey): return self.load_connection_file() transport = self.transport ip = self.ip if transport != 'tcp': self.log.error("Can only use ssh tunnels with TCP sockets, not %s", transport) sys.exit(-1) if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = ip ip = localhost() # build connection dict for tunnels: info = dict(ip=ip, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port) self.log.info("Forwarding connections to %s via %s" % (ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: self.ip = localhost() try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except: # even catch KeyboardInterrupt self.log.error("Could not setup tunnels", exc_info=True) self.exit(1) self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports cf = self.connection_file base, ext = os.path.splitext(cf) base = os.path.basename(base) self.connection_file = os.path.basename(base) + '-ssh' + ext self.log.info("To connect another client via this tunnel, use:") self.log.info("--existing %s" % self.connection_file) def _new_connection_file(self): cf = '' while not cf: # we don't need a 128b id to distinguish kernels, use more readable # 48b node segment (12 hex chars). Users running more than 32k simultaneous # kernels can subclass. ident = str(uuid.uuid4()).split('-')[-1] cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident) # only keep if it's actually new. Protect against unlikely collision # in 48b random search space cf = cf if not os.path.exists(cf) else '' return cf def init_kernel_manager(self): # Don't let Qt or ZMQ swallow KeyboardInterupts. if self.existing: self.kernel_manager = None return signal.signal(signal.SIGINT, signal.SIG_DFL) # Create a KernelManager and start a kernel. try: self.kernel_manager = self.kernel_manager_class( ip=self.ip, session=self.session, transport=self.transport, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, connection_file=self.connection_file, kernel_name=self.kernel_name, parent=self, ipython_dir=self.ipython_dir, ) except NoSuchKernel: self.log.critical("Could not find kernel %s", self.kernel_name) self.exit(1) self.kernel_manager.client_factory = self.kernel_client_class # FIXME: remove special treatment of IPython kernels kwargs = {} if self.kernel_manager.ipython_kernel: kwargs['extra_arguments'] = self.kernel_argv self.kernel_manager.start_kernel(**kwargs) atexit.register(self.kernel_manager.cleanup_ipc_files) if self.sshserver: # ssh, write new connection file self.kernel_manager.write_connection_file() # in case KM defaults / ssh writing changes things: km = self.kernel_manager self.shell_port = km.shell_port self.iopub_port = km.iopub_port self.stdin_port = km.stdin_port self.hb_port = km.hb_port self.connection_file = km.connection_file atexit.register(self.kernel_manager.cleanup_connection_file) def init_kernel_client(self): if self.kernel_manager is not None: self.kernel_client = self.kernel_manager.client() else: self.kernel_client = self.kernel_client_class( session=self.session, ip=self.ip, transport=self.transport, shell_port=self.shell_port, iopub_port=self.iopub_port, stdin_port=self.stdin_port, hb_port=self.hb_port, connection_file=self.connection_file, parent=self, ) self.kernel_client.start_channels() def initialize(self, argv=None): """ Classes which mix this class in should call: IPythonConsoleApp.initialize(self,argv) """ self.init_connection_file() self.init_ssh() self.init_kernel_manager() self.init_kernel_client()
class MappingKernelManager(MultiKernelManager): """A KernelManager that handles notebook mapping and HTTP error handling""" def _kernel_manager_class_default(self): return "IPython.kernel.ioloop.IOLoopKernelManager" kernel_argv = List(Unicode) root_dir = Unicode(config=True) def _root_dir_default(self): try: return self.parent.notebook_dir except AttributeError: return getcwd() def _root_dir_changed(self, name, old, new): """Do a bit of validation of the root dir.""" if not os.path.isabs(new): # If we receive a non-absolute path, make it absolute. self.root_dir = os.path.abspath(new) return if not os.path.exists(new) or not os.path.isdir(new): raise TraitError("kernel root dir %r is not a directory" % new) #------------------------------------------------------------------------- # Methods for managing kernels and sessions #------------------------------------------------------------------------- def _handle_kernel_died(self, kernel_id): """notice that a kernel died""" self.log.warn("Kernel %s died, removing from map.", kernel_id) self.remove_kernel(kernel_id) def cwd_for_path(self, path): """Turn API path into absolute OS path.""" os_path = to_os_path(path, self.root_dir) # in the case of notebooks and kernels not being on the same filesystem, # walk up to root_dir if the paths don't exist while not os.path.isdir(os_path) and os_path != self.root_dir: os_path = os.path.dirname(os_path) return os_path def start_kernel(self, kernel_id=None, path=None, kernel_name='python', **kwargs): """Start a kernel for a session and return its kernel_id. Parameters ---------- kernel_id : uuid The uuid to associate the new kernel with. If this is not None, this kernel will be persistent whenever it is requested. path : API path The API path (unicode, '/' delimited) for the cwd. Will be transformed to an OS path relative to root_dir. kernel_name : str The name identifying which kernel spec to launch. This is ignored if an existing kernel is returned, but it may be checked in the future. """ if kernel_id is None: if path is not None: kwargs['cwd'] = self.cwd_for_path(path) kernel_id = super(MappingKernelManager, self).start_kernel(kernel_name=kernel_name, **kwargs) self.log.info("Kernel started: %s" % kernel_id) self.log.debug("Kernel args: %r" % kwargs) # register callback for failed auto-restart self.add_restart_callback( kernel_id, lambda: self._handle_kernel_died(kernel_id), 'dead', ) else: self._check_kernel_id(kernel_id) self.log.info("Using existing kernel: %s" % kernel_id) return kernel_id def shutdown_kernel(self, kernel_id, now=False): """Shutdown a kernel by kernel_id""" self._check_kernel_id(kernel_id) super(MappingKernelManager, self).shutdown_kernel(kernel_id, now=now) def kernel_model(self, kernel_id): """Return a dictionary of kernel information described in the JSON standard model.""" self._check_kernel_id(kernel_id) model = {"id": kernel_id, "name": self._kernels[kernel_id].kernel_name} return model def list_kernels(self): """Returns a list of kernel_id's of kernels running.""" kernels = [] kernel_ids = super(MappingKernelManager, self).list_kernel_ids() for kernel_id in kernel_ids: model = self.kernel_model(kernel_id) kernels.append(model) return kernels # override _check_kernel_id to raise 404 instead of KeyError def _check_kernel_id(self, kernel_id): """Check a that a kernel_id exists and raise 404 if not.""" if kernel_id not in self: raise web.HTTPError(404, u'Kernel does not exist: %s' % kernel_id)
class InProcessKernel(Kernel): #------------------------------------------------------------------------- # InProcessKernel interface #------------------------------------------------------------------------- # The frontends connected to this kernel. frontends = List( Instance('IPython.kernel.inprocess.client.InProcessKernelClient')) # The GUI environment that the kernel is running under. This need not be # specified for the normal operation for the kernel, but is required for # IPython's GUI support (including pylab). The default is 'inline' because # it is safe under all GUI toolkits. gui = Enum(('tk', 'gtk', 'wx', 'qt', 'qt4', 'inline'), default_value='inline') raw_input_str = Any() stdout = Any() stderr = Any() #------------------------------------------------------------------------- # Kernel interface #------------------------------------------------------------------------- shell_class = Type() shell_streams = List() control_stream = Any() iopub_socket = Instance(DummySocket, ()) stdin_socket = Instance(DummySocket, ()) def __init__(self, **traits): # When an InteractiveShell is instantiated by our base class, it binds # the current values of sys.stdout and sys.stderr. with self._redirected_io(): super(InProcessKernel, self).__init__(**traits) self.iopub_socket.on_trait_change(self._io_dispatch, 'message_sent') self.shell.kernel = self def execute_request(self, stream, ident, parent): """ Override for temporary IO redirection. """ with self._redirected_io(): super(InProcessKernel, self).execute_request(stream, ident, parent) def start(self): """ Override registration of dispatchers for streams. """ self.shell.exit_now = False def _abort_queue(self, stream): """ The in-process kernel doesn't abort requests. """ pass def _raw_input(self, prompt, ident, parent): # Flush output before making the request. self.raw_input_str = None sys.stderr.flush() sys.stdout.flush() # Send the input request. content = json_clean(dict(prompt=prompt)) msg = self.session.msg('input_request', content, parent) for frontend in self.frontends: if frontend.session.session == parent['header']['session']: frontend.stdin_channel.call_handlers(msg) break else: logging.error('No frontend found for raw_input request') return str() # Await a response. while self.raw_input_str is None: frontend.stdin_channel.process_events() return self.raw_input_str #------------------------------------------------------------------------- # Protected interface #------------------------------------------------------------------------- @contextmanager def _redirected_io(self): """ Temporarily redirect IO to the kernel. """ sys_stdout, sys_stderr = sys.stdout, sys.stderr sys.stdout, sys.stderr = self.stdout, self.stderr yield sys.stdout, sys.stderr = sys_stdout, sys_stderr #------ Trait change handlers -------------------------------------------- def _io_dispatch(self): """ Called when a message is sent to the IO socket. """ ident, msg = self.session.recv(self.iopub_socket, copy=False) for frontend in self.frontends: frontend.iopub_channel.call_handlers(msg) #------ Trait initializers ----------------------------------------------- def _log_default(self): return logging.getLogger(__name__) def _session_default(self): from IPython.kernel.zmq.session import Session return Session(parent=self) def _shell_class_default(self): return InProcessInteractiveShell def _stdout_default(self): from IPython.kernel.zmq.iostream import OutStream return OutStream(self.session, self.iopub_socket, 'stdout', pipe=False) def _stderr_default(self): from IPython.kernel.zmq.iostream import OutStream return OutStream(self.session, self.iopub_socket, 'stderr', pipe=False)
class TemplateExporter(Exporter): """ Exports notebooks into other file formats. Uses Jinja 2 templating engine to output new formats. Inherit from this class if you are creating a new template type along with new filters/preprocessors. If the filters/ preprocessors provided by default suffice, there is no need to inherit from this class. Instead, override the template_file and file_extension traits via a config file. {filters} """ # finish the docstring __doc__ = __doc__.format(filters='- ' + '\n - '.join(default_filters.keys())) template_file = Unicode(u'default', config=True, help="Name of the template file to use") def _template_file_changed(self, name, old, new): if new == 'default': self.template_file = self.default_template else: self.template_file = new self.template = None self._load_template() default_template = Unicode(u'') template = Any() environment = Any() template_path = List(['.'], config=True) def _template_path_changed(self, name, old, new): self._load_template() default_template_path = Unicode( os.path.join("..", "templates"), help="Path where the template files are located.") template_skeleton_path = Unicode( os.path.join("..", "templates", "skeleton"), help="Path where the template skeleton files are located.") #Jinja block definitions jinja_comment_block_start = Unicode("", config=True) jinja_comment_block_end = Unicode("", config=True) jinja_variable_block_start = Unicode("", config=True) jinja_variable_block_end = Unicode("", config=True) jinja_logic_block_start = Unicode("", config=True) jinja_logic_block_end = Unicode("", config=True) #Extension that the template files use. template_extension = Unicode(".tpl", config=True) filters = Dict( config=True, help="""Dictionary of filters, by name and namespace, to add to the Jinja environment.""") def __init__(self, config=None, extra_loaders=None, **kw): """ Public constructor Parameters ---------- config : config User configuration instance. extra_loaders : list[of Jinja Loaders] ordered list of Jinja loader to find templates. Will be tried in order before the default FileSystem ones. template : str (optional, kw arg) Template to use when exporting. """ super(TemplateExporter, self).__init__(config=config, **kw) #Init self._init_template() self._init_environment(extra_loaders=extra_loaders) self._init_preprocessors() self._init_filters() def _load_template(self): """Load the Jinja template object from the template file This is a no-op if the template attribute is already defined, or the Jinja environment is not setup yet. This is triggered by various trait changes that would change the template. """ if self.template is not None: return # called too early, do nothing if self.environment is None: return # Try different template names during conversion. First try to load the # template by name with extension added, then try loading the template # as if the name is explicitly specified, then try the name as a # 'flavor', and lastly just try to load the template by module name. module_name = self.__module__.rsplit('.', 1)[-1] try_names = [] if self.template_file: try_names.extend([ self.template_file + self.template_extension, self.template_file, module_name + '_' + self.template_file + self.template_extension, ]) try_names.append(module_name + self.template_extension) for try_name in try_names: self.log.debug("Attempting to load template %s", try_name) try: self.template = self.environment.get_template(try_name) except (TemplateNotFound, IOError): pass except Exception as e: self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True) else: self.log.info("Loaded template %s", try_name) break def from_notebook_node(self, nb, resources=None, **kw): """ Convert a notebook from a notebook node instance. Parameters ---------- nb : Notebook node resources : dict (**kw) of additional resources that can be accessed read/write by preprocessors and filters. """ nb_copy, resources = super(TemplateExporter, self).from_notebook_node( nb, resources, **kw) self._load_template() if self.template is not None: output = self.template.render(nb=nb_copy, resources=resources) else: raise IOError('template file "%s" could not be found' % self.template_file) return output, resources def register_filter(self, name, jinja_filter): """ Register a filter. A filter is a function that accepts and acts on one string. The filters are accesible within the Jinja templating engine. Parameters ---------- name : str name to give the filter in the Jinja engine filter : filter """ if jinja_filter is None: raise TypeError('filter') isclass = isinstance(jinja_filter, type) constructed = not isclass #Handle filter's registration based on it's type if constructed and isinstance(jinja_filter, py3compat.string_types): #filter is a string, import the namespace and recursively call #this register_filter method filter_cls = import_item(jinja_filter) return self.register_filter(name, filter_cls) if constructed and hasattr(jinja_filter, '__call__'): #filter is a function, no need to construct it. self.environment.filters[name] = jinja_filter return jinja_filter elif isclass and isinstance(jinja_filter, MetaHasTraits): #filter is configurable. Make sure to pass in new default for #the enabled flag if one was specified. filter_instance = jinja_filter(parent=self) self.register_filter(name, filter_instance) elif isclass: #filter is not configurable, construct it filter_instance = jinja_filter() self.register_filter(name, filter_instance) else: #filter is an instance of something without a __call__ #attribute. raise TypeError('filter') def _init_template(self): """ Make sure a template name is specified. If one isn't specified, try to build one from the information we know. """ self._template_file_changed('template_file', self.template_file, self.template_file) def _init_environment(self, extra_loaders=None): """ Create the Jinja templating environment. """ here = os.path.dirname(os.path.realpath(__file__)) loaders = [] if extra_loaders: loaders.extend(extra_loaders) paths = self.template_path paths.extend([ os.path.join(here, self.default_template_path), os.path.join(here, self.template_skeleton_path) ]) loaders.append(FileSystemLoader(paths)) self.environment = Environment(loader=ChoiceLoader(loaders), extensions=JINJA_EXTENSIONS) #Set special Jinja2 syntax that will not conflict with latex. if self.jinja_logic_block_start: self.environment.block_start_string = self.jinja_logic_block_start if self.jinja_logic_block_end: self.environment.block_end_string = self.jinja_logic_block_end if self.jinja_variable_block_start: self.environment.variable_start_string = self.jinja_variable_block_start if self.jinja_variable_block_end: self.environment.variable_end_string = self.jinja_variable_block_end if self.jinja_comment_block_start: self.environment.comment_start_string = self.jinja_comment_block_start if self.jinja_comment_block_end: self.environment.comment_end_string = self.jinja_comment_block_end def _init_filters(self): """ Register all of the filters required for the exporter. """ #Add default filters to the Jinja2 environment for key, value in default_filters.items(): self.register_filter(key, value) #Load user filters. Overwrite existing filters if need be. if self.filters: for key, user_filter in self.filters.items(): self.register_filter(key, user_filter)
class AutoHandler(PrefilterHandler): handler_name = Unicode('auto') esc_strings = List([ESC_PAREN, ESC_QUOTE, ESC_QUOTE2]) def handle(self, line_info): """Handle lines which can be auto-executed, quoting if requested.""" line = line_info.line ifun = line_info.ifun the_rest = line_info.the_rest pre = line_info.pre esc = line_info.esc continue_prompt = line_info.continue_prompt obj = line_info.ofind(self.shell)['obj'] #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun,the_rest) # dbg # This should only be active for single-line input! if continue_prompt: return line force_auto = isinstance(obj, IPyAutocall) # User objects sometimes raise exceptions on attribute access other # than AttributeError (we've seen it in the past), so it's safest to be # ultra-conservative here and catch all. try: auto_rewrite = obj.rewrite except Exception: auto_rewrite = True if esc == ESC_QUOTE: # Auto-quote splitting on whitespace newcmd = '%s("%s")' % (ifun,'", "'.join(the_rest.split()) ) elif esc == ESC_QUOTE2: # Auto-quote whole string newcmd = '%s("%s")' % (ifun,the_rest) elif esc == ESC_PAREN: newcmd = '%s(%s)' % (ifun,",".join(the_rest.split())) else: # Auto-paren. if force_auto: # Don't rewrite if it is already a call. do_rewrite = not the_rest.startswith('(') else: if not the_rest: # We only apply it to argument-less calls if the autocall # parameter is set to 2. do_rewrite = (self.shell.autocall >= 2) elif the_rest.startswith('[') and hasattr(obj, '__getitem__'): # Don't autocall in this case: item access for an object # which is BOTH callable and implements __getitem__. do_rewrite = False else: do_rewrite = True # Figure out the rewritten command if do_rewrite: if the_rest.endswith(';'): newcmd = '%s(%s);' % (ifun.rstrip(),the_rest[:-1]) else: newcmd = '%s(%s)' % (ifun.rstrip(), the_rest) else: normal_handler = self.prefilter_manager.get_handler_by_name('normal') return normal_handler.handle(line_info) # Display the rewritten call if auto_rewrite: self.shell.auto_rewrite_input(newcmd) return newcmd
class Kernel(Configurable): #--------------------------------------------------------------------------- # Kernel interface #--------------------------------------------------------------------------- # attribute to override with a GUI eventloop = Any(None) def _eventloop_changed(self, name, old, new): """schedule call to eventloop from IOLoop""" loop = ioloop.IOLoop.instance() loop.add_timeout(time.time()+0.1, self.enter_eventloop) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') shell_class = Type(ZMQInteractiveShell) session = Instance(Session) profile_dir = Instance('IPython.core.profiledir.ProfileDir') shell_streams = List() control_stream = Instance(ZMQStream) iopub_socket = Instance(zmq.Socket) stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) user_module = Any() def _user_module_changed(self, name, old, new): if self.shell is not None: self.shell.user_module = new user_ns = Dict(default_value=None) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new self.shell.init_user_ns() # identities: int_id = Integer(-1) ident = Unicode() def _ident_default(self): return unicode(uuid.uuid4()) # Private interface # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for # clients. # Units are in seconds. The minimum zmq latency on local host is probably # ~150 microseconds, set this to 500us for now. We may need to increase it # a little if it's not enough after more interactive testing. _execute_sleep = Float(0.0005, config=True) # Frequency of the kernel's event loop. # Units are in seconds, kernel subclasses for GUI toolkits may need to # adapt to milliseconds. _poll_interval = Float(0.05, config=True) # If the shutdown was requested over the network, we leave here the # necessary reply message so it can be sent by our registered atexit # handler. This ensures that the reply is only sent to clients truly at # the end of our shutdown process (which happens after the underlying # IPython shell's own shutdown). _shutdown_message = None # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() # A reference to the Python builtin 'raw_input' function. # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3) _sys_raw_input = Any() # set of aborted msg_ids aborted = Set() def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) # Initialize the InteractiveShell subclass self.shell = self.shell_class.instance(config=self.config, profile_dir = self.profile_dir, user_module = self.user_module, user_ns = self.user_ns, ) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket self.shell.displayhook.topic = self._topic('pyout') self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket self.shell.data_pub.session = self.session self.shell.data_pub.pub_socket = self.iopub_socket # TMP - hack while developing self.shell._reply_content = None # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'object_info_request', 'history_request', 'kernel_info_request', 'connect_request', 'shutdown_request', 'apply_request', ] self.shell_handlers = {} for msg_type in msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) control_msg_types = msg_types + [ 'clear_request', 'abort_request' ] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) def dispatch_control(self, msg): """dispatch control requests""" idents,msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Control Message", exc_info=True) return self.log.debug("Control received: %s", msg) header = msg['header'] msg_id = header['msg_id'] msg_type = header['msg_type'] handler = self.control_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: try: handler(self.control_stream, idents, msg) except Exception: self.log.error("Exception in control handler:", exc_info=True) def dispatch_shell(self, stream, msg): """dispatch shell requests""" # flush control requests first if self.control_stream: self.control_stream.flush() idents,msg = self.session.feed_identities(msg, copy=False) try: msg = self.session.unserialize(msg, content=True, copy=False) except: self.log.error("Invalid Message", exc_info=True) return header = msg['header'] msg_id = header['msg_id'] msg_type = msg['header']['msg_type'] # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) if msg_id in self.aborted: self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status' : 'aborted'} md = {'engine' : self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) return handler = self.shell_handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: # ensure default_int_handler during handler call sig = signal(SIGINT, default_int_handler) try: handler(stream, idents, msg) except Exception: self.log.error("Exception in message handler:", exc_info=True) finally: signal(SIGINT, sig) def enter_eventloop(self): """enter eventloop""" self.log.info("entering eventloop") # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: try: self.eventloop(self) except KeyboardInterrupt: # Ctrl-C shouldn't crash the kernel self.log.error("KeyboardInterrupt caught in kernel") continue else: # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break self.log.info("exiting eventloop") def start(self): """register dispatchers for streams""" self.shell.exit_now = False if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) def make_dispatcher(stream): def dispatcher(msg): return self.dispatch_shell(stream, msg) return dispatcher for s in self.shell_streams: s.on_recv(make_dispatcher(s), copy=False) def do_one_iteration(self): """step eventloop just once""" if self.control_stream: self.control_stream.flush() for stream in self.shell_streams: # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) def record_ports(self, ports): """Record the ports that this kernel is using. The creator of the Kernel instance must call this methods if they want the :meth:`connect_request` method to return the port numbers. """ self._recorded_ports = ports #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- def _make_metadata(self, other=None): """init metadata dict, for execute/apply_reply""" new_md = { 'dependencies_met' : True, 'engine' : self.ident, 'started': datetime.now(), } if other: new_md.update(other) return new_md def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" self.session.send(self.iopub_socket, u'pyin', {u'code':code, u'execution_count': execution_count}, parent=parent, ident=self._topic('pyin') ) def _publish_status(self, status, parent=None): """send status (busy/idle) on IOPub""" self.session.send(self.iopub_socket, u'status', {u'execution_state': status}, parent=parent, ident=self._topic('status'), ) def execute_request(self, stream, ident, parent): """handle an execute_request""" self._publish_status(u'busy', parent) try: content = parent[u'content'] code = content[u'code'] silent = content[u'silent'] store_history = content.get(u'store_history', not silent) except: self.log.error("Got bad msg: ") self.log.error("%s", parent) return md = self._make_metadata(parent['metadata']) shell = self.shell # we'll need this a lot here # Replace raw_input. Note that is not sufficient to replace # raw_input in the user namespace. if content.get('allow_stdin', False): raw_input = lambda prompt='': self._raw_input(prompt, ident, parent) else: raw_input = lambda prompt='' : self._no_raw_input() if py3compat.PY3: self._sys_raw_input = __builtin__.input __builtin__.input = raw_input else: self._sys_raw_input = __builtin__.raw_input __builtin__.raw_input = raw_input # Set the parent message of the display hook and out streams. shell.displayhook.set_parent(parent) shell.display_pub.set_parent(parent) shell.data_pub.set_parent(parent) try: sys.stdout.set_parent(parent) except AttributeError: pass try: sys.stderr.set_parent(parent) except AttributeError: pass # Re-broadcast our input for the benefit of listening clients, and # start computing output if not silent: self._publish_pyin(code, parent, shell.execution_count) reply_content = {} try: # FIXME: the shell calls the exception handler itself. shell.run_cell(code, store_history=store_history, silent=silent) except: status = u'error' # FIXME: this code right now isn't being used yet by default, # because the run_cell() call above directly fires off exception # reporting. This code, therefore, is only active in the scenario # where runlines itself has an unhandled exception. We need to # uniformize this, for all exception construction to come from a # single location in the codbase. etype, evalue, tb = sys.exc_info() tb_list = traceback.format_exception(etype, evalue, tb) reply_content.update(shell._showtraceback(etype, evalue, tb_list)) else: status = u'ok' finally: # Restore raw_input. if py3compat.PY3: __builtin__.input = self._sys_raw_input else: __builtin__.raw_input = self._sys_raw_input reply_content[u'status'] = status # Return the execution counter so clients can display prompts reply_content['execution_count'] = shell.execution_count - 1 # FIXME - fish exception info out of shell, possibly left there by # runlines. We'll need to clean up this logic later. if shell._reply_content is not None: reply_content.update(shell._reply_content) e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute') reply_content['engine_info'] = e_info # reset after use shell._reply_content = None if 'traceback' in reply_content: self.log.info("Exception in execute request:\n%s", '\n'.join(reply_content['traceback'])) # At this point, we can tell whether the main code execution succeeded # or not. If it did, we proceed to evaluate user_variables/expressions if reply_content['status'] == 'ok': reply_content[u'user_variables'] = \ shell.user_variables(content.get(u'user_variables', [])) reply_content[u'user_expressions'] = \ shell.user_expressions(content.get(u'user_expressions', {})) else: # If there was an error, don't even try to compute variables or # expressions reply_content[u'user_variables'] = {} reply_content[u'user_expressions'] = {} # Payloads should be retrieved regardless of outcome, so we can both # recover partial output (that could have been generated early in a # block, before an error) and clear the payload system always. reply_content[u'payload'] = shell.payload_manager.read_payload() # Be agressive about clearing the payload because we don't want # it to sit in memory until the next execute_request comes in. shell.payload_manager.clear_payload() # Flush output before sending the reply. sys.stdout.flush() sys.stderr.flush() # FIXME: on rare occasions, the flush doesn't seem to make it to the # clients... This seems to mitigate the problem, but we definitely need # to better understand what's going on. if self._execute_sleep: time.sleep(self._execute_sleep) # Send the reply. reply_content = json_clean(reply_content) md['status'] = reply_content['status'] if reply_content['status'] == 'error' and \ reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, metadata=md, ident=ident) self.log.debug("%s", reply_msg) if not silent and reply_msg['content']['status'] == u'error': self._abort_queues() self._publish_status(u'idle', parent) def complete_request(self, stream, ident, parent): txt, matches = self._complete(parent) matches = {'matches' : matches, 'matched_text' : txt, 'status' : 'ok'} matches = json_clean(matches) completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) def object_info_request(self, stream, ident, parent): content = parent['content'] object_info = self.shell.object_inspect(content['oname'], detail_level = content.get('detail_level', 0) ) # Before we send this object over, we scrub it for JSON usage oinfo = json_clean(object_info) msg = self.session.send(stream, 'object_info_reply', oinfo, parent, ident) self.log.debug("%s", msg) def history_request(self, stream, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with # unicode keys before Python 2.6.5. hist_access_type = parent['content']['hist_access_type'] raw = parent['content']['raw'] output = parent['content']['output'] if hist_access_type == 'tail': n = parent['content']['n'] hist = self.shell.history_manager.get_tail(n, raw=raw, output=output, include_latest=True) elif hist_access_type == 'range': session = parent['content']['session'] start = parent['content']['start'] stop = parent['content']['stop'] hist = self.shell.history_manager.get_range(session, start, stop, raw=raw, output=output) elif hist_access_type == 'search': n = parent['content'].get('n') unique = parent['content'].get('unique', False) pattern = parent['content']['pattern'] hist = self.shell.history_manager.search( pattern, raw=raw, output=output, n=n, unique=unique) else: hist = [] hist = list(hist) content = {'history' : hist} content = json_clean(content) msg = self.session.send(stream, 'history_reply', content, parent, ident) self.log.debug("Sending history reply with %i entries", len(hist)) def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) def kernel_info_request(self, stream, ident, parent): vinfo = { 'protocol_version': protocol_version, 'ipython_version': ipython_version, 'language_version': language_version, 'language': 'python', } msg = self.session.send(stream, 'kernel_info_reply', vinfo, parent, ident) self.log.debug("%s", msg) def shutdown_request(self, stream, ident, parent): self.shell.exit_now = True content = dict(status='ok') content.update(parent['content']) self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', content, parent ) self._at_shutdown() # call sys.exit after a short delay loop = ioloop.IOLoop.instance() loop.add_timeout(time.time()+0.1, loop.stop) #--------------------------------------------------------------------------- # Engine methods #--------------------------------------------------------------------------- def apply_request(self, stream, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] except: self.log.error("Got bad msg: %s", parent, exc_info=True) return self._publish_status(u'busy', parent) # Set the parent message of the display hook and out streams. shell = self.shell shell.displayhook.set_parent(parent) shell.display_pub.set_parent(parent) shell.data_pub.set_parent(parent) try: sys.stdout.set_parent(parent) except AttributeError: pass try: sys.stderr.set_parent(parent) except AttributeError: pass # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_socket.send(pyin_msg) # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) md = self._make_metadata(parent['metadata']) try: working = shell.user_ns prefix = "_"+str(msg_id).replace("-","")+"_" f,args,kwargs = unpack_apply_message(bufs, working, copy=False) fname = getattr(f, '__name__', 'f') fname = prefix+"f" argname = prefix+"args" kwargname = prefix+"kwargs" resultname = prefix+"result" ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } # print ns working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: exec code in shell.user_global_ns, shell.user_ns result = working.get(resultname) finally: for key in ns.iterkeys(): working.pop(key) result_buf = serialize_object(result, buffer_threshold=self.session.buffer_threshold, item_threshold=self.session.item_threshold, ) except: # invoke IPython traceback formatting shell.showtraceback() # FIXME - fish exception info out of shell, possibly left there by # run_code. We'll need to clean up this logic later. reply_content = {} if shell._reply_content is not None: reply_content.update(shell._reply_content) e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') reply_content['engine_info'] = e_info # reset after use shell._reply_content = None self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent, ident=self._topic('pyerr')) self.log.info("Exception in apply request:\n%s", '\n'.join(reply_content['traceback'])) result_buf = [] if reply_content['ename'] == 'UnmetDependency': md['dependencies_met'] = False else: reply_content = {'status' : 'ok'} # put 'ok'/'error' status in header, for scheduler introspection: md['status'] = reply_content['status'] # flush i/o sys.stdout.flush() sys.stderr.flush() reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident,buffers=result_buf, metadata=md) self._publish_status(u'idle', parent) #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- def abort_request(self, stream, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, basestring): msg_ids = [msg_ids] if not msg_ids: self.abort_queues() for mid in msg_ids: self.aborted.add(str(mid)) content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" self.shell.reset(False) msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content = dict(status='ok')) #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- def _wrap_exception(self, method=None): # import here, because _wrap_exception is only used in parallel, # and parallel has higher min pyzmq version from IPython.parallel.error import wrap_exception e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method) content = wrap_exception(e_info) return content def _topic(self, topic): """prefixed topic for IOPub messages""" if self.int_id >= 0: base = "engine.%i" % self.int_id else: base = "kernel.%s" % self.ident return py3compat.cast_bytes("%s.%s" % (base, topic)) def _abort_queues(self): for stream in self.shell_streams: if stream: self._abort_queue(stream) def _abort_queue(self, stream): poller = zmq.Poller() poller.register(stream.socket, zmq.POLLIN) while True: idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return self.log.info("Aborting:") self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' status = {'status' : 'aborted'} md = {'engine' : self.ident} md.update(status) reply_msg = self.session.send(stream, reply_type, metadata=md, content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. poller.poll(50) def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support stdin.""" raise StdinNotImplementedError("raw_input was called, but this " "frontend does not support stdin.") def _raw_input(self, prompt, ident, parent): # Flush output before making the request. sys.stderr.flush() sys.stdout.flush() # Send the input request. content = json_clean(dict(prompt=prompt)) self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) # Await a response. while True: try: ident, reply = self.session.recv(self.stdin_socket, 0) except Exception: self.log.warn("Invalid Message:", exc_info=True) else: break try: value = reply['content']['value'] except: self.log.error("Got bad raw_input reply: ") self.log.error("%s", parent) value = '' if value == '\x04': # EOF raise EOFError return value def _complete(self, msg): c = msg['content'] try: cpos = int(c['cursor_pos']) except: # If we don't get something that we can convert to an integer, at # least attempt the completion guessing the cursor is at the end of # the text, if there's any, and otherwise of the line cpos = len(c['text']) if cpos==0: cpos = len(c['line']) return self.shell.complete(c['text'], c['line'], cpos) def _at_shutdown(self): """Actions taken at shutdown by the kernel, called by python's atexit. """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
class IPControllerApp(BaseParallelApplication): name = u'ipcontroller' description = _description examples = _examples classes = [ ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB ] + real_dbs # change default to True auto_create = Bool( True, config=True, help="""Whether to create profile dir if it doesn't exist.""") reuse_files = Bool(False, config=True, help="""Whether to reuse existing json connection files. If False, connection files will be removed on a clean exit. """) restore_engines = Bool(False, config=True, help="""Reload engine state from JSON file """) ssh_server = Unicode( u'', config=True, help="""ssh url for clients to use when connecting to the Controller processes. It should be of the form: [user@]server[:port]. The Controller's listening addresses must be accessible from the ssh server""", ) engine_ssh_server = Unicode( u'', config=True, help="""ssh url for engines to use when connecting to the Controller processes. It should be of the form: [user@]server[:port]. The Controller's listening addresses must be accessible from the ssh server""", ) location = Unicode( u'', config=True, help= """The external IP or domain name of the Controller, used for disambiguating engine and client connections.""", ) import_statements = List( [], config=True, help= "import statements to be run at startup. Necessary in some environments" ) use_threads = Bool( False, config=True, help='Use threads instead of processes for the schedulers', ) engine_json_file = Unicode( 'ipcontroller-engine.json', config=True, help="JSON filename where engine connection info will be stored.") client_json_file = Unicode( 'ipcontroller-client.json', config=True, help="JSON filename where client connection info will be stored.") def _cluster_id_changed(self, name, old, new): super(IPControllerApp, self)._cluster_id_changed(name, old, new) self.engine_json_file = "%s-engine.json" % self.name self.client_json_file = "%s-client.json" % self.name # internal children = List() mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') def _use_threads_changed(self, name, old, new): self.mq_class = 'zmq.devices.%sMonitoredQueue' % ('Thread' if new else 'Process') write_connection_files = Bool( True, help="""Whether to write connection files to disk. True in all cases other than runs with `reuse_files=True` *after the first* """) aliases = Dict(aliases) flags = Dict(flags) def save_connection_dict(self, fname, cdict): """save a connection dict to json file.""" c = self.config url = cdict['registration'] location = cdict['location'] if not location: if public_ips(): location = public_ips()[-1] else: self.log.warn( "Could not identify this machine's IP, assuming %s." " You may need to specify '--location=<external_ip_address>' to help" " IPython decide when to connect via loopback." % localhost()) location = localhost() cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) self.log.info("writing connection info to %s", fname) with open(fname, 'w') as f: f.write(json.dumps(cdict, indent=2)) os.chmod(fname, stat.S_IRUSR | stat.S_IWUSR) def load_config_from_json(self): """load config from existing json connector files.""" c = self.config self.log.debug("loading config from JSON") # load engine config fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file) self.log.info("loading connection info from %s", fname) with open(fname) as f: ecfg = json.loads(f.read()) # json gives unicode, Session.key wants bytes c.Session.key = ecfg['key'].encode('ascii') xport, ip = ecfg['interface'].split('://') c.HubFactory.engine_ip = ip c.HubFactory.engine_transport = xport self.location = ecfg['location'] if not self.engine_ssh_server: self.engine_ssh_server = ecfg['ssh'] # load client config fname = os.path.join(self.profile_dir.security_dir, self.client_json_file) self.log.info("loading connection info from %s", fname) with open(fname) as f: ccfg = json.loads(f.read()) for key in ('key', 'registration', 'pack', 'unpack', 'signature_scheme'): assert ccfg[key] == ecfg[ key], "mismatch between engine and client info: %r" % key xport, addr = ccfg['interface'].split('://') c.HubFactory.client_transport = xport c.HubFactory.client_ip = ip if not self.ssh_server: self.ssh_server = ccfg['ssh'] # load port config: c.HubFactory.regport = ecfg['registration'] c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong']) c.HubFactory.control = (ccfg['control'], ecfg['control']) c.HubFactory.mux = (ccfg['mux'], ecfg['mux']) c.HubFactory.task = (ccfg['task'], ecfg['task']) c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub']) c.HubFactory.notifier_port = ccfg['notification'] def cleanup_connection_files(self): if self.reuse_files: self.log.debug("leaving JSON connection files for reuse") return self.log.debug("cleaning up JSON connection files") for f in (self.client_json_file, self.engine_json_file): f = os.path.join(self.profile_dir.security_dir, f) try: os.remove(f) except Exception as e: self.log.error("Failed to cleanup connection file: %s", e) else: self.log.debug(u"removed %s", f) def load_secondary_config(self): """secondary config, loading from JSON and setting defaults""" if self.reuse_files: try: self.load_config_from_json() except (AssertionError, IOError) as e: self.log.error("Could not load config from JSON: %s" % e) else: # successfully loaded config from JSON, and reuse=True # no need to wite back the same file self.write_connection_files = False self.log.debug("Config changed") self.log.debug(repr(self.config)) def init_hub(self): c = self.config self.do_import_statements() try: self.factory = HubFactory(config=c, log=self.log) # self.start_logging() self.factory.init_hub() except TraitError: raise except Exception: self.log.error("Couldn't construct the Controller", exc_info=True) self.exit(1) if self.write_connection_files: # save to new json config files f = self.factory base = { 'key': f.session.key.decode('ascii'), 'location': self.location, 'pack': f.session.packer, 'unpack': f.session.unpacker, 'signature_scheme': f.session.signature_scheme, } cdict = {'ssh': self.ssh_server} cdict.update(f.client_info) cdict.update(base) self.save_connection_dict(self.client_json_file, cdict) edict = {'ssh': self.engine_ssh_server} edict.update(f.engine_info) edict.update(base) self.save_connection_dict(self.engine_json_file, edict) fname = "engines%s.json" % self.cluster_id self.factory.hub.engine_state_file = os.path.join( self.profile_dir.log_dir, fname) if self.restore_engines: self.factory.hub._load_engine_state() # load key into config so other sessions in this process (TaskScheduler) # have the same value self.config.Session.key = self.factory.session.key def init_schedulers(self): children = self.children mq = import_item(str(self.mq_class)) f = self.factory ident = f.session.bsession # disambiguate url, in case of * monitor_url = disambiguate_url(f.monitor_url) # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url # IOPub relay (in a Process) q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A', b'iopub') q.bind_in(f.client_url('iopub')) q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") q.bind_out(f.engine_url('iopub')) q.setsockopt_out(zmq.SUBSCRIBE, b'') q.connect_mon(monitor_url) q.daemon = True children.append(q) # Multiplexer Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') q.bind_in(f.client_url('mux')) q.setsockopt_in(zmq.IDENTITY, b'mux_in') q.bind_out(f.engine_url('mux')) q.setsockopt_out(zmq.IDENTITY, b'mux_out') q.connect_mon(monitor_url) q.daemon = True children.append(q) # Control Queue (in a Process) q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') q.bind_in(f.client_url('control')) q.setsockopt_in(zmq.IDENTITY, b'control_in') q.bind_out(f.engine_url('control')) q.setsockopt_out(zmq.IDENTITY, b'control_out') q.connect_mon(monitor_url) q.daemon = True children.append(q) if 'TaskScheduler.scheme_name' in self.config: scheme = self.config.TaskScheduler.scheme_name else: scheme = TaskScheduler.scheme_name.get_default_value() # Task Queue (in a Process) if scheme == 'pure': self.log.warn("task::using pure DEALER Task scheduler") q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') # q.setsockopt_out(zmq.HWM, hub.hwm) q.bind_in(f.client_url('task')) q.setsockopt_in(zmq.IDENTITY, b'task_in') q.bind_out(f.engine_url('task')) q.setsockopt_out(zmq.IDENTITY, b'task_out') q.connect_mon(monitor_url) q.daemon = True children.append(q) elif scheme == 'none': self.log.warn("task::using no Task scheduler") else: self.log.info("task::using Python %s Task scheduler" % scheme) sargs = ( f.client_url('task'), f.engine_url('task'), monitor_url, disambiguate_url(f.client_url('notification')), disambiguate_url(f.client_url('registration')), ) kwargs = dict(logname='scheduler', loglevel=self.log_level, log_url=self.log_url, config=dict(self.config)) if 'Process' in self.mq_class: # run the Python scheduler in a Process q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) q.daemon = True children.append(q) else: # single-threaded Controller kwargs['in_thread'] = True launch_scheduler(*sargs, **kwargs) # set unlimited HWM for all relay devices if hasattr(zmq, 'SNDHWM'): q = children[0] q.setsockopt_in(zmq.RCVHWM, 0) q.setsockopt_out(zmq.SNDHWM, 0) for q in children[1:]: if not hasattr(q, 'setsockopt_in'): continue q.setsockopt_in(zmq.SNDHWM, 0) q.setsockopt_in(zmq.RCVHWM, 0) q.setsockopt_out(zmq.SNDHWM, 0) q.setsockopt_out(zmq.RCVHWM, 0) q.setsockopt_mon(zmq.SNDHWM, 0) def terminate_children(self): child_procs = [] for child in self.children: if isinstance(child, ProcessMonitoredQueue): child_procs.append(child.launcher) elif isinstance(child, Process): child_procs.append(child) if child_procs: self.log.critical("terminating children...") for child in child_procs: try: child.terminate() except OSError: # already dead pass def handle_signal(self, sig, frame): self.log.critical("Received signal %i, shutting down", sig) self.terminate_children() self.loop.stop() def init_signal(self): for sig in (SIGINT, SIGABRT, SIGTERM): signal(sig, self.handle_signal) def do_import_statements(self): statements = self.import_statements for s in statements: try: self.log.msg("Executing statement: '%s'" % s) exec(s, globals(), locals()) except: self.log.msg("Error running statement: %s" % s) def forward_logging(self): if self.log_url: self.log.info("Forwarding logging to %s" % self.log_url) context = zmq.Context.instance() lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = PUBHandler(lsock) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler) @catch_config_error def initialize(self, argv=None): super(IPControllerApp, self).initialize(argv) self.forward_logging() self.load_secondary_config() self.init_hub() self.init_schedulers() def start(self): # Start the subprocesses: self.factory.start() # children must be started before signals are setup, # otherwise signal-handling will fire multiple times for child in self.children: child.start() self.init_signal() self.write_pid_file(overwrite=True) try: self.factory.loop.start() except KeyboardInterrupt: self.log.critical("Interrupted, Exiting...\n") finally: self.cleanup_connection_files()
class HistoryManager(HistoryAccessor): """A class to organize all history-related functionality in one place. """ # Public interface # An instance of the IPython shell we are attached to shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') # Lists to hold processed and raw history. These start with a blank entry # so that we can index them starting from 1 input_hist_parsed = List([""]) input_hist_raw = List([""]) # A list of directories visited during session dir_hist = List() def _dir_hist_default(self): try: return [py3compat.getcwd()] except OSError: return [] # A dict of output history, keyed with ints from the shell's # execution count. output_hist = Dict() # The text/plain repr of outputs. output_hist_reprs = Dict() # The number of the current session in the history database session_number = Integer() db_log_output = Bool( False, config=True, help="Should the history database include output? (default: no)") db_cache_size = Integer( 0, config=True, help= "Write to database every x commands (higher values save disk access & power).\n" "Values of 1 or less effectively disable caching.") # The input and output caches db_input_cache = List() db_output_cache = List() # History saving in separate thread save_thread = Instance('IPython.core.history.HistorySavingThread') try: # Event is a function returning an instance of _Event... save_flag = Instance(threading._Event) except AttributeError: # ...until Python 3.3, when it's a class. save_flag = Instance(threading.Event) # Private interface # Variables used to store the three last inputs from the user. On each new # history update, we populate the user's namespace with these, shifted as # necessary. _i00 = Unicode(u'') _i = Unicode(u'') _ii = Unicode(u'') _iii = Unicode(u'') # A regex matching all forms of the exit command, so that we don't store # them in the history (it's annoying to rewind the first entry and land on # an exit call). _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") def __init__(self, shell=None, config=None, **traits): """Create a new history manager associated with a shell instance. """ # We need a pointer back to the shell for various tasks. super(HistoryManager, self).__init__(shell=shell, config=config, **traits) self.save_flag = threading.Event() self.db_input_cache_lock = threading.Lock() self.db_output_cache_lock = threading.Lock() if self.enabled and self.hist_file != ':memory:': self.save_thread = HistorySavingThread(self) self.save_thread.start() self.new_session() def _get_hist_file_name(self, profile=None): """Get default history file name based on the Shell's profile. The profile parameter is ignored, but must exist for compatibility with the parent class.""" profile_dir = self.shell.profile_dir.location return os.path.join(profile_dir, 'history.sqlite') @needs_sqlite def new_session(self, conn=None): """Get a new session number.""" if conn is None: conn = self.db with conn: cur = conn.execute( """INSERT INTO sessions VALUES (NULL, ?, NULL, NULL, "") """, (datetime.datetime.now(), )) self.session_number = cur.lastrowid def end_session(self): """Close the database session, filling in the end time and line count.""" self.writeout_cache() with self.db: self.db.execute( """UPDATE sessions SET end=?, num_cmds=? WHERE session==?""", (datetime.datetime.now(), len(self.input_hist_parsed) - 1, self.session_number)) self.session_number = 0 def name_session(self, name): """Give the current session a name in the history database.""" with self.db: self.db.execute("UPDATE sessions SET remark=? WHERE session==?", (name, self.session_number)) def reset(self, new_session=True): """Clear the session history, releasing all object references, and optionally open a new session.""" self.output_hist.clear() # The directory history can't be completely empty self.dir_hist[:] = [py3compat.getcwd()] if new_session: if self.session_number: self.end_session() self.input_hist_parsed[:] = [""] self.input_hist_raw[:] = [""] self.new_session() # ------------------------------ # Methods for retrieving history # ------------------------------ def get_session_info(self, session=0): """Get info about a session. Parameters ---------- session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is the previous session. Returns ------- session_id : int Session ID number start : datetime Timestamp for the start of the session. end : datetime Timestamp for the end of the session, or None if IPython crashed. num_cmds : int Number of commands run, or None if IPython crashed. remark : unicode A manually set description. """ if session <= 0: session += self.session_number return super(HistoryManager, self).get_session_info(session=session) def _get_range_session(self, start=1, stop=None, raw=True, output=False): """Get input and output history from the current session. Called by get_range, and takes similar parameters.""" input_hist = self.input_hist_raw if raw else self.input_hist_parsed n = len(input_hist) if start < 0: start += n if not stop or (stop > n): stop = n elif stop < 0: stop += n for i in range(start, stop): if output: line = (input_hist[i], self.output_hist_reprs.get(i)) else: line = input_hist[i] yield (0, i, line) def get_range(self, session=0, start=1, stop=None, raw=True, output=False): """Retrieve input by session. Parameters ---------- session : int Session number to retrieve. The current session is 0, and negative numbers count back from current session, so -1 is previous session. start : int First line to retrieve. stop : int End of line range (excluded from output itself). If None, retrieve to the end of the session. raw : bool If True, return untranslated input output : bool If True, attempt to include output. This will be 'real' Python objects for the current session, or text reprs from previous sessions if db_log_output was enabled at the time. Where no output is found, None is used. Returns ------- entries An iterator over the desired lines. Each line is a 3-tuple, either (session, line, input) if output is False, or (session, line, (input, output)) if output is True. """ if session <= 0: session += self.session_number if session == self.session_number: # Current session return self._get_range_session(start, stop, raw, output) return super(HistoryManager, self).get_range(session, start, stop, raw, output) ## ---------------------------- ## Methods for storing history: ## ---------------------------- def store_inputs(self, line_num, source, source_raw=None): """Store source and raw input in history and create input cache variables ``_i*``. Parameters ---------- line_num : int The prompt number of this input. source : str Python input. source_raw : str, optional If given, this is the raw input without any IPython transformations applied to it. If not given, ``source`` is used. """ if source_raw is None: source_raw = source source = source.rstrip('\n') source_raw = source_raw.rstrip('\n') # do not store exit/quit commands if self._exit_re.match(source_raw.strip()): return self.input_hist_parsed.append(source) self.input_hist_raw.append(source_raw) with self.db_input_cache_lock: self.db_input_cache.append((line_num, source, source_raw)) # Trigger to flush cache and write to DB. if len(self.db_input_cache) >= self.db_cache_size: self.save_flag.set() # update the auto _i variables self._iii = self._ii self._ii = self._i self._i = self._i00 self._i00 = source_raw # hackish access to user namespace to create _i1,_i2... dynamically new_i = '_i%s' % line_num to_main = { '_i': self._i, '_ii': self._ii, '_iii': self._iii, new_i: self._i00 } if self.shell is not None: self.shell.push(to_main, interactive=False) def store_output(self, line_num): """If database output logging is enabled, this saves all the outputs from the indicated prompt number to the database. It's called by run_cell after code has been executed. Parameters ---------- line_num : int The line number from which to save outputs """ if (not self.db_log_output) or (line_num not in self.output_hist_reprs): return output = self.output_hist_reprs[line_num] with self.db_output_cache_lock: self.db_output_cache.append((line_num, output)) if self.db_cache_size <= 1: self.save_flag.set() def _writeout_input_cache(self, conn): with conn: for line in self.db_input_cache: conn.execute("INSERT INTO history VALUES (?, ?, ?, ?)", (self.session_number, ) + line) def _writeout_output_cache(self, conn): with conn: for line in self.db_output_cache: conn.execute("INSERT INTO output_history VALUES (?, ?, ?)", (self.session_number, ) + line) @needs_sqlite def writeout_cache(self, conn=None): """Write any entries in the cache to the database.""" if conn is None: conn = self.db with self.db_input_cache_lock: try: self._writeout_input_cache(conn) except sqlite3.IntegrityError: self.new_session(conn) print("ERROR! Session/line number was not unique in", "database. History logging moved to new session", self.session_number) try: # Try writing to the new session. If this fails, don't # recurse self._writeout_input_cache(conn) except sqlite3.IntegrityError: pass finally: self.db_input_cache = [] with self.db_output_cache_lock: try: self._writeout_output_cache(conn) except sqlite3.IntegrityError: print("!! Session/line number for output was not unique", "in database. Output will not be stored.") finally: self.db_output_cache = []
class SubmitApp(BaseIPythonApplication): name = Unicode(u'nbgrader-submit') description = Unicode(u'Submit a completed assignment') aliases = aliases flags = flags examples = examples student = Unicode(os.environ['USER']) timestamp = Unicode(str(datetime.datetime.now())) assignment_directory = Unicode('.', config=True, help=dedent(""" The directory containing the assignment to be submitted. """)) assignment_name = Unicode('', config=True, help=dedent(""" The name of the assignment. Defaults to the name of the assignment directory. """)) submissions_directory = Unicode("{}/.nbgrader/submissions".format( os.environ['HOME']), config=True, help=dedent(""" The directory where the submission will be saved. """)) ignore = List([".ipynb_checkpoints", "*.pyc", "__pycache__"], config=True, help=dedent(""" List of file names or file globs to be ignored when creating the submission. """)) def _ipython_dir_default(self): d = os.path.join(os.environ["HOME"], ".nbgrader") self._ipython_dir_changed('ipython_dir', d, d) return d # The classes added here determine how configuration will be documented classes = List() def _classes_default(self): """This has to be in a method, for TerminalIPythonApp to be available.""" return [ProfileDir] def _log_level_default(self): return logging.INFO @catch_config_error def initialize(self, argv=None): if not os.path.exists(self.ipython_dir): self.log.warning("Creating nbgrader directory: {}".format( self.ipython_dir)) os.mkdir(self.ipython_dir) if not os.path.exists(self.submissions_directory): os.makedirs(self.submissions_directory) super(SubmitApp, self).initialize(argv) self.stage_default_config_file() self.init_assignment_root() if self.assignment_name == '': self.assignment_name = os.path.basename(self.assignment_directory) def init_assignment_root(self): # Specifying notebooks on the command-line overrides (rather than adds) # the notebook list if self.extra_args: patterns = self.extra_args else: patterns = [self.assignment_directory] if len(patterns) == 0: pass elif len(patterns) == 1: self.assignment_directory = patterns[0] else: raise ValueError("You must specify the name of a directory") self.assignment_directory = os.path.abspath(self.assignment_directory) if not os.path.isdir(self.assignment_directory): raise ValueError("Path is not a directory: {}".format( self.assignment_directory)) def _is_ignored(self, filename): dirname = os.path.dirname(filename) for expr in self.ignore: globs = glob.glob(os.path.join(dirname, expr)) if filename in globs: self.log.debug("Ignoring file: {}".format(filename)) return True return False def make_temp_copy(self): """Copy the submission to a temporary directory. Returns the path to the temporary copy of the submission.""" # copy everything to a temporary directory pth = os.path.join(self.tmpdir, self.assignment_name) shutil.copytree(self.assignment_directory, pth) os.chdir(self.tmpdir) # get the user name, write it to file with open(os.path.join(self.assignment_name, "user.txt"), "w") as fh: fh.write(self.student) # save the submission time with open(os.path.join(self.assignment_name, "timestamp.txt"), "w") as fh: fh.write(self.timestamp) return pth def make_archive(self, path_to_submission): """Make a tarball of the submission. Returns the path to the created archive.""" root, submission = os.path.split(path_to_submission) os.chdir(root) archive = os.path.join(self.tmpdir, "{}.tar.gz".format(self.assignment_name)) tf = tarfile.open(archive, "w:gz") for (dirname, dirnames, filenames) in os.walk(submission): if self._is_ignored(dirname): continue for filename in filenames: pth = os.path.join(dirname, filename) if not self._is_ignored(pth): self.log.debug("Adding '{}' to submission".format(pth)) tf.add(pth) tf.close() return archive def submit(self, path_to_tarball): """Submit the assignment.""" archive = "{}.tar.gz".format(self.assignment_name) target = os.path.join(self.submissions_directory, archive) shutil.copy(path_to_tarball, target) return target def start(self): super(SubmitApp, self).start() self.tmpdir = tempfile.mkdtemp() try: path_to_copy = self.make_temp_copy() path_to_tarball = self.make_archive(path_to_copy) path_to_submission = self.submit(path_to_tarball) except: raise else: self.log.debug("Saved to '{}'".format(path_to_submission)) self.log.info("'{}' submitted by {} at {}".format( self.assignment_name, self.student, self.timestamp)) finally: shutil.rmtree(self.tmpdir)